From 8fc6a3aaef5f38e0aa4ae5d14311fe3c30643204 Mon Sep 17 00:00:00 2001 From: Honey Date: Mon, 17 Aug 2026 12:28:32 +0200 Subject: [PATCH] feat(backend): PO token provider for YouTube downloads; drop the cookies approach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the cookies-file stopgap (needs a throwaway account + manual refresh) with a self-hosted PO token provider — the cookie-less way past YouTube's datacenter-IP block (bot check + media 403). - docker-compose: new bgutil-provider service (brainicism/bgutil-ytdlp-pot-provider) + install the bgutil-ytdlp-pot-provider plugin in the backend + POT_PROVIDER_URL env + depends_on. - ytdlp.js / server.js: when POT_PROVIDER_URL is set, pass --extractor-args youtubepot-bgutilhttp:base_url=... to the download cascade, the YTM search probe and the importer resolve. Empty → omitted (unchanged behaviour). - Removed the cookiesFile config + --cookies wiring so there's no dead code. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 14 +++++++++----- backend/lib/config.js | 15 ++++++--------- backend/lib/ytdlp.js | 16 +++++++++------- backend/novasonic-server.js | 5 ++--- docker-compose.yml | 15 ++++++++++++++- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb1249..28a0ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,11 +88,15 @@ to Android at parity with web, and fixes a batch of same-name-artist and player as avatar chips, matching the Android app (previously plain text). ### Fixed -- **Downloads — YouTube bot check ("Sign in to confirm you're not a bot")** — YouTube now blocks - yt-dlp from datacenter IPs. The backend passes a **cookies file** to every yt-dlp call when present - (`YTDLP_COOKIES`, default `/data/youtube-cookies.txt`): drop a Netscape `cookies.txt` from a - logged-in (ideally throwaway) account into the data volume and downloads/search/resolve - authenticate again. Absent → yt-dlp runs exactly as before (no behaviour change). +- **Downloads — YouTube datacenter-IP block (bot check + media 403)** — YouTube now blocks yt-dlp + from datacenter IPs (first "Sign in to confirm you're not a bot", then `HTTP 403` on the media + stream, which needs a **PO token** yt-dlp can't compute itself). Added a **`bgutil-provider`** + service (Docker) + the `bgutil-ytdlp-pot-provider` plugin in the backend; yt-dlp fetches PO tokens + from it — no cookies or account needed. Guarded by `POT_PROVIDER_URL` (empty → yt-dlp runs exactly + as before). See `docker-compose.yml`. +- **Downloads — a failed direct-link download reported success** — the direct-URL path returned + `success` even when no audio was produced (e.g. the YouTube 403), so the job showed "Fertig!" and + the UI (web + Android) said the track was added. It now reports failure with a clear message. - **Artist pin — search & similar artists (Web + Android)** — a pinned artist now shows the correct Deezer photo in library search (not the wrong same-name person's), and its "Similar Artists" use Deezer's **related** for the pinned artist instead of Last.fm's list (which was keyed to the diff --git a/backend/lib/config.js b/backend/lib/config.js index a01db4c..dd6945e 100644 --- a/backend/lib/config.js +++ b/backend/lib/config.js @@ -1,21 +1,18 @@ -const path = require('path'); - -const dataPath = process.env.DATA_PATH || '/data'; - const CONFIG = { port: process.env.PORT || 3700, musicPath: process.env.MUSIC_PATH || '/music', - dataPath, + dataPath: process.env.DATA_PATH || '/data', ytdlpBin: process.env.YTDLP_BIN || 'yt-dlp', audioFmt: process.env.AUDIO_FMT || 'mp3', audioQuality: process.env.AUDIO_QUALITY || '0', maxParallel: parseInt(process.env.MAX_PARALLEL || '2', 10), downloadBackend: (process.env.DOWNLOAD_BACKEND || 'ytdlp').toLowerCase(), ticketmasterKey: process.env.TICKETMASTER_API_KEY || '', - // YouTube bot-blocks datacenter IPs ("Sign in to confirm you're not a bot"). Passing cookies from - // a logged-in (ideally throwaway) account fixes it. Drop a Netscape-format cookies.txt at this path - // (default: in the data volume) and every yt-dlp call uses it. No file → yt-dlp runs as before. - cookiesFile: process.env.YTDLP_COOKIES || path.join(dataPath, 'youtube-cookies.txt'), + // YouTube 403s media downloads from datacenter IPs without a PO ("Proof of Origin") token, which + // yt-dlp can't compute itself. The bgutil-ytdlp-pot-provider plugin fetches tokens from a small + // provider service; set its URL here (e.g. http://bgutil-provider:4416). Empty → no PO token used + // (yt-dlp runs exactly as before). See docker-compose.yml. + potProviderUrl: process.env.POT_PROVIDER_URL || '', }; module.exports = CONFIG; diff --git a/backend/lib/ytdlp.js b/backend/lib/ytdlp.js index c8daefc..f5bebd7 100644 --- a/backend/lib/ytdlp.js +++ b/backend/lib/ytdlp.js @@ -134,11 +134,13 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb // YTM ranking toward a same-named artist's full discography. const query = `${ytmArtist} ${searchTitle}`; - // YouTube bot-blocks datacenter IPs ("Sign in to confirm you're not a bot"). If a cookies file is - // present, pass it so yt-dlp authenticates. Checked at call time so dropping the file in takes - // effect without a code change; absent → no flag → unchanged behaviour. - const cookieArgs = () => - (CONFIG.cookiesFile && fs.existsSync(CONFIG.cookiesFile)) ? ['--cookies', CONFIG.cookiesFile] : []; + // YouTube 403s media downloads from datacenter IPs without a PO token. When a provider URL is + // configured, point the bgutil-ytdlp-pot-provider plugin at it so yt-dlp fetches a token. Empty + // config → no arg → unchanged behaviour. + const potArgs = () => + CONFIG.potProviderUrl + ? ['--extractor-args', `youtubepot-bgutilhttp:base_url=${CONFIG.potProviderUrl}`] + : []; const baseArgs = [ '--extract-audio', @@ -154,7 +156,7 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb // "Art Track" / topic uploads are reachable. The android_vr client returns // "This video is not available" for many YTM tracks that play fine in the browser. '--extractor-args', 'youtube:player_client=web_music,default', - ...cookieArgs(), + ...potArgs(), ]; const spawnYtdlp = (args) => new Promise((resolve, reject) => { @@ -227,7 +229,7 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb // generic YouTube search and returns unrelated videos instead of the // native Songs shelf (where the actual artist track lives). '--extractor-args', 'youtube:player_client=web_music,default', - ...cookieArgs(), + ...potArgs(), ], { env: { ...process.env, PYTHONUNBUFFERED: '1' } }); proc.stdout.on('data', d => { const text = d.toString(); diff --git a/backend/novasonic-server.js b/backend/novasonic-server.js index 35a27a7..c945989 100644 --- a/backend/novasonic-server.js +++ b/backend/novasonic-server.js @@ -533,9 +533,8 @@ app.post('/api/youtube/resolve', async (req, res) => { execFile(CONFIG.ytdlpBin, [ '--dump-json', '--flat-playlist', '--no-warnings', '--playlist-items', '1-50', - // Cookies (if present) let yt-dlp past YouTube's datacenter-IP bot check. Domain-scoped, so - // harmless for non-YouTube links. No file → runs as before. - ...((CONFIG.cookiesFile && fs.existsSync(CONFIG.cookiesFile)) ? ['--cookies', CONFIG.cookiesFile] : []), + // PO token provider (if configured) to get past YouTube's datacenter-IP block. Empty → omitted. + ...(CONFIG.potProviderUrl ? ['--extractor-args', `youtubepot-bgutilhttp:base_url=${CONFIG.potProviderUrl}`] : []), url, ], { timeout: 45000, maxBuffer: 20 * 1024 * 1024 }, (err, out, stderr) => { if (err && !out?.trim()) return reject(new Error(stderr?.trim() || err.message)); diff --git a/docker-compose.yml b/docker-compose.yml index 3f1e681..a478498 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,15 +33,28 @@ services: - MAX_PARALLEL=${MAX_PARALLEL:-2} - DOWNLOAD_BACKEND=${DOWNLOAD_BACKEND:-ytdlp} - TICKETMASTER_API_KEY=${TICKETMASTER_API_KEY} + # PO token provider (bgutil) — lets yt-dlp download from datacenter IPs without cookies/account. + - POT_PROVIDER_URL=http://bgutil-provider:4416 + depends_on: + - bgutil-provider command: > sh -c " apt-get update && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg curl git && rm -rf /var/lib/apt/lists/* && curl -sL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && chmod a+rx /usr/local/bin/yt-dlp && - pip install --break-system-packages spotapi 'git+https://github.com/TzurSoffer/spotify-downloader' && + pip install --break-system-packages spotapi 'git+https://github.com/TzurSoffer/spotify-downloader' bgutil-ytdlp-pot-provider && mkdir -p /deps && cd /deps && npm install express kuromoji && cd /app && NODE_PATH=/deps/node_modules node novasonic-server.js " + # ── PO token provider (bgutil) — mints YouTube "Proof of Origin" tokens so yt-dlp can download + # from a datacenter IP without cookies or an account. Paired with the bgutil-ytdlp-pot-provider + # plugin installed in the backend above. Port 4416 is internal only (no reverse proxy needed). ── + bgutil-provider: + image: brainicism/bgutil-ytdlp-pot-provider:latest + container_name: bgutil-provider + restart: unless-stopped + init: true + # ── Scrobble service (isolated timestamped play-log → stats + Wrapped) ──────── novasonic-scrobble: image: node:22-slim