feat(backend): PO token provider for YouTube downloads; drop the cookies approach
Replace the cookies-file stopgap (needs a throwaway account + manual refresh) with a self-hosted PO token provider — the cookie-less way past YouTube's datacenter-IP block (bot check + media 403). - docker-compose: new bgutil-provider service (brainicism/bgutil-ytdlp-pot-provider) + install the bgutil-ytdlp-pot-provider plugin in the backend + POT_PROVIDER_URL env + depends_on. - ytdlp.js / server.js: when POT_PROVIDER_URL is set, pass --extractor-args youtubepot-bgutilhttp:base_url=... to the download cascade, the YTM search probe and the importer resolve. Empty → omitted (unchanged behaviour). - Removed the cookiesFile config + --cookies wiring so there's no dead code. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
HoneyandClaude Opus 4.8 committed
1 parent
ef019b9b9d
commit
8fc6a3aaef
5 files changed
+40
-25
No files matched your search
+9
-5
@@ -88,11 +88,15 @@ to Android at parity with web, and fixes a batch of same-name-artist and player
|
||||
as avatar chips, matching the Android app (previously plain text).
|
||||
|
||||
### Fixed
|
||||
- **Downloads — YouTube bot check ("Sign in to confirm you're not a bot")** — YouTube now blocks
|
||||
yt-dlp from datacenter IPs. The backend passes a **cookies file** to every yt-dlp call when present
|
||||
(`YTDLP_COOKIES`, default `/data/youtube-cookies.txt`): drop a Netscape `cookies.txt` from a
|
||||
logged-in (ideally throwaway) account into the data volume and downloads/search/resolve
|
||||
authenticate again. Absent → yt-dlp runs exactly as before (no behaviour change).
|
||||
- **Downloads — YouTube datacenter-IP block (bot check + media 403)** — YouTube now blocks yt-dlp
|
||||
from datacenter IPs (first "Sign in to confirm you're not a bot", then `HTTP 403` on the media
|
||||
stream, which needs a **PO token** yt-dlp can't compute itself). Added a **`bgutil-provider`**
|
||||
service (Docker) + the `bgutil-ytdlp-pot-provider` plugin in the backend; yt-dlp fetches PO tokens
|
||||
from it — no cookies or account needed. Guarded by `POT_PROVIDER_URL` (empty → yt-dlp runs exactly
|
||||
as before). See `docker-compose.yml`.
|
||||
- **Downloads — a failed direct-link download reported success** — the direct-URL path returned
|
||||
`success` even when no audio was produced (e.g. the YouTube 403), so the job showed "Fertig!" and
|
||||
the UI (web + Android) said the track was added. It now reports failure with a clear message.
|
||||
- **Artist pin — search & similar artists (Web + Android)** — a pinned artist now shows the correct
|
||||
Deezer photo in library search (not the wrong same-name person's), and its "Similar Artists" use
|
||||
Deezer's **related** for the pinned artist instead of Last.fm's list (which was keyed to the
|
||||
|
||||
@@ -1,21 +1,18 @@
|
||||
const path = require('path');
|
||||
|
||||
const dataPath = process.env.DATA_PATH || '/data';
|
||||
|
||||
const CONFIG = {
|
||||
port: process.env.PORT || 3700,
|
||||
musicPath: process.env.MUSIC_PATH || '/music',
|
||||
dataPath,
|
||||
dataPath: process.env.DATA_PATH || '/data',
|
||||
ytdlpBin: process.env.YTDLP_BIN || 'yt-dlp',
|
||||
audioFmt: process.env.AUDIO_FMT || 'mp3',
|
||||
audioQuality: process.env.AUDIO_QUALITY || '0',
|
||||
maxParallel: parseInt(process.env.MAX_PARALLEL || '2', 10),
|
||||
downloadBackend: (process.env.DOWNLOAD_BACKEND || 'ytdlp').toLowerCase(),
|
||||
ticketmasterKey: process.env.TICKETMASTER_API_KEY || '',
|
||||
// YouTube bot-blocks datacenter IPs ("Sign in to confirm you're not a bot"). Passing cookies from
|
||||
// a logged-in (ideally throwaway) account fixes it. Drop a Netscape-format cookies.txt at this path
|
||||
// (default: in the data volume) and every yt-dlp call uses it. No file → yt-dlp runs as before.
|
||||
cookiesFile: process.env.YTDLP_COOKIES || path.join(dataPath, 'youtube-cookies.txt'),
|
||||
// YouTube 403s media downloads from datacenter IPs without a PO ("Proof of Origin") token, which
|
||||
// yt-dlp can't compute itself. The bgutil-ytdlp-pot-provider plugin fetches tokens from a small
|
||||
// provider service; set its URL here (e.g. http://bgutil-provider:4416). Empty → no PO token used
|
||||
// (yt-dlp runs exactly as before). See docker-compose.yml.
|
||||
potProviderUrl: process.env.POT_PROVIDER_URL || '',
|
||||
};
|
||||
|
||||
module.exports = CONFIG;
|
||||
@@ -134,11 +134,13 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
|
||||
// YTM ranking toward a same-named artist's full discography.
|
||||
const query = `${ytmArtist} ${searchTitle}`;
|
||||
|
||||
// YouTube bot-blocks datacenter IPs ("Sign in to confirm you're not a bot"). If a cookies file is
|
||||
// present, pass it so yt-dlp authenticates. Checked at call time so dropping the file in takes
|
||||
// effect without a code change; absent → no flag → unchanged behaviour.
|
||||
const cookieArgs = () =>
|
||||
(CONFIG.cookiesFile && fs.existsSync(CONFIG.cookiesFile)) ? ['--cookies', CONFIG.cookiesFile] : [];
|
||||
// YouTube 403s media downloads from datacenter IPs without a PO token. When a provider URL is
|
||||
// configured, point the bgutil-ytdlp-pot-provider plugin at it so yt-dlp fetches a token. Empty
|
||||
// config → no arg → unchanged behaviour.
|
||||
const potArgs = () =>
|
||||
CONFIG.potProviderUrl
|
||||
? ['--extractor-args', `youtubepot-bgutilhttp:base_url=${CONFIG.potProviderUrl}`]
|
||||
: [];
|
||||
|
||||
const baseArgs = [
|
||||
'--extract-audio',
|
||||
@@ -154,7 +156,7 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
|
||||
// "Art Track" / topic uploads are reachable. The android_vr client returns
|
||||
// "This video is not available" for many YTM tracks that play fine in the browser.
|
||||
'--extractor-args', 'youtube:player_client=web_music,default',
|
||||
...cookieArgs(),
|
||||
...potArgs(),
|
||||
];
|
||||
|
||||
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
|
||||
@@ -227,7 +229,7 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
|
||||
// generic YouTube search and returns unrelated videos instead of the
|
||||
// native Songs shelf (where the actual artist track lives).
|
||||
'--extractor-args', 'youtube:player_client=web_music,default',
|
||||
...cookieArgs(),
|
||||
...potArgs(),
|
||||
], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
||||
proc.stdout.on('data', d => {
|
||||
const text = d.toString();
|
||||
|
||||
@@ -533,9 +533,8 @@ app.post('/api/youtube/resolve', async (req, res) => {
|
||||
execFile(CONFIG.ytdlpBin, [
|
||||
'--dump-json', '--flat-playlist', '--no-warnings',
|
||||
'--playlist-items', '1-50',
|
||||
// Cookies (if present) let yt-dlp past YouTube's datacenter-IP bot check. Domain-scoped, so
|
||||
// harmless for non-YouTube links. No file → runs as before.
|
||||
...((CONFIG.cookiesFile && fs.existsSync(CONFIG.cookiesFile)) ? ['--cookies', CONFIG.cookiesFile] : []),
|
||||
// PO token provider (if configured) to get past YouTube's datacenter-IP block. Empty → omitted.
|
||||
...(CONFIG.potProviderUrl ? ['--extractor-args', `youtubepot-bgutilhttp:base_url=${CONFIG.potProviderUrl}`] : []),
|
||||
url,
|
||||
], { timeout: 45000, maxBuffer: 20 * 1024 * 1024 }, (err, out, stderr) => {
|
||||
if (err && !out?.trim()) return reject(new Error(stderr?.trim() || err.message));
|
||||
|
||||
+14
-1
@@ -33,15 +33,28 @@ services:
|
||||
- MAX_PARALLEL=${MAX_PARALLEL:-2}
|
||||
- DOWNLOAD_BACKEND=${DOWNLOAD_BACKEND:-ytdlp}
|
||||
- TICKETMASTER_API_KEY=${TICKETMASTER_API_KEY}
|
||||
# PO token provider (bgutil) — lets yt-dlp download from datacenter IPs without cookies/account.
|
||||
- POT_PROVIDER_URL=http://bgutil-provider:4416
|
||||
depends_on:
|
||||
- bgutil-provider
|
||||
command: >
|
||||
sh -c "
|
||||
apt-get update && apt-get install -y --no-install-recommends python3 python3-pip ffmpeg curl git && rm -rf /var/lib/apt/lists/* &&
|
||||
curl -sL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && chmod a+rx /usr/local/bin/yt-dlp &&
|
||||
pip install --break-system-packages spotapi 'git+https://github.com/TzurSoffer/spotify-downloader' &&
|
||||
pip install --break-system-packages spotapi 'git+https://github.com/TzurSoffer/spotify-downloader' bgutil-ytdlp-pot-provider &&
|
||||
mkdir -p /deps && cd /deps && npm install express kuromoji &&
|
||||
cd /app && NODE_PATH=/deps/node_modules node novasonic-server.js
|
||||
"
|
||||
|
||||
# ── PO token provider (bgutil) — mints YouTube "Proof of Origin" tokens so yt-dlp can download
|
||||
# from a datacenter IP without cookies or an account. Paired with the bgutil-ytdlp-pot-provider
|
||||
# plugin installed in the backend above. Port 4416 is internal only (no reverse proxy needed). ──
|
||||
bgutil-provider:
|
||||
image: brainicism/bgutil-ytdlp-pot-provider:latest
|
||||
container_name: bgutil-provider
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
|
||||
# ── Scrobble service (isolated timestamped play-log → stats + Wrapped) ────────
|
||||
novasonic-scrobble:
|
||||
image: node:22-slim
|
||||
|
||||
Reference in new issue
Block a user