diff --git a/.env.example b/.env.example index 25a2550..ea891d1 100644 --- a/.env.example +++ b/.env.example @@ -5,9 +5,20 @@ NAVIDROME_URL=https://music.yourdomain.com BACKEND_URL=https://novasonic-api.yourdomain.com # Absolute path to your Navidrome music library on the host -# This exact path is used inside the container — no subdirectory is added MUSIC_PATH=/srv/navidrome/music -AUDIO_FMT=mp3 # mp3 | flac | opus | best -AUDIO_QUALITY=0 # 0 = best quality +AUDIO_FMT=mp3 # mp3 | flac | opus | best (used by yt-dlp) +AUDIO_QUALITY=0 # 0 = best quality (used by yt-dlp) MAX_PARALLEL=2 + +# Download backend: ytdlp | squid | auto +# ytdlp - always use yt-dlp +# squid - always use squid.wtf (Qobuz) +# auto - try squid.wtf first, fall back to yt-dlp if not found (default) +DOWNLOAD_BACKEND=auto + +# Quality for squid.wtf / Qobuz downloads +# mp3 - MP3 320 kbps +# flac - FLAC 16-bit / 44.1 kHz +# hi-res - FLAC 24-bit / 96 kHz +SQUID_QUALITY=mp3 diff --git a/backend/lib/config.js b/backend/lib/config.js new file mode 100644 index 0000000..c6c951c --- /dev/null +++ b/backend/lib/config.js @@ -0,0 +1,14 @@ +const CONFIG = { + port: process.env.PORT || 3700, + musicPath: process.env.MUSIC_PATH || '/music', + ytdlpBin: process.env.YTDLP_BIN || 'yt-dlp', + audioFmt: process.env.AUDIO_FMT || 'mp3', + audioQuality: process.env.AUDIO_QUALITY || '0', + maxParallel: parseInt(process.env.MAX_PARALLEL || '2', 10), + // 'ytdlp' | 'squid' | 'auto' (squid first, yt-dlp fallback) + downloadBackend: (process.env.DOWNLOAD_BACKEND || 'auto').toLowerCase(), + // squid.wtf quality: 'mp3' | 'flac' | 'hi-res' + squidQuality: (process.env.SQUID_QUALITY || 'mp3').toLowerCase(), +}; + +module.exports = CONFIG; diff --git a/backend/lib/downloader.js b/backend/lib/downloader.js new file mode 100644 index 0000000..ec71aab --- /dev/null +++ b/backend/lib/downloader.js @@ -0,0 +1,60 @@ +const CONFIG = require('./config'); +const { log, updateJob } = require('./jobs'); +const ytdlp = require('./ytdlp'); +const squid = require('./squid'); + +// Route a single track download to the correct backend, with optional fallback. +async function downloadTrack(opts) { + const backend = CONFIG.downloadBackend; + + if (backend === 'ytdlp') return ytdlp.downloadTrack(opts); + if (backend === 'squid') return squid.downloadTrack(opts); + + // 'auto': squid first, yt-dlp as fallback + try { + const result = await squid.downloadTrack(opts); + if (result !== null) return result; + // null = not found on Qobuz — fall through to yt-dlp + log(opts.jobId, '⚠️ squid.wtf: nicht gefunden — Fallback zu yt-dlp'); + } catch (e) { + log(opts.jobId, `⚠️ squid.wtf Fehler — Fallback zu yt-dlp: ${e.message}`); + } + return ytdlp.downloadTrack(opts); +} + +// Download all tracks of an album sequentially, updating job progress. +async function downloadAlbumTracks({ jobId, artist, albumTitle, tracks, artworkUrl }) { + const total = tracks.length; + log(jobId, `📀 Album "${albumTitle}" — ${total} Tracks`); + let done = 0, skipped = 0; + + for (const track of tracks) { + try { + const result = await downloadTrack({ + jobId, + artist: track.artistName || artist, + albumArtist: artist, + title: track.trackName || track.title, + album: albumTitle, + year: track.releaseDate ? new Date(track.releaseDate).getFullYear() : undefined, + trackNum: track.trackNumber || undefined, + totalTracks: total, + discNum: track.discNumber || undefined, + discTotal: track.discCount || undefined, + compilation: track.collectionType === 'Compilation' || undefined, + genre: track.primaryGenreName || undefined, + artworkUrl: artworkUrl || track.artworkUrl100, + explicit: track.trackExplicitness === 'explicit', + outputDir: CONFIG.musicPath, + durationMs: track.trackTimeMillis || undefined, + }); + if (result?.skipped) skipped++; else done++; + updateJob(jobId, { progress: Math.round(((done + skipped) / total) * 90) }); + } catch (e) { + log(jobId, `⚠️ Fehler bei "${track.trackName || track.title}": ${e.message}`); + } + } + return { done, skipped }; +} + +module.exports = { downloadTrack, downloadAlbumTracks }; diff --git a/backend/lib/jobs.js b/backend/lib/jobs.js new file mode 100644 index 0000000..ee0a780 --- /dev/null +++ b/backend/lib/jobs.js @@ -0,0 +1,14 @@ +const jobs = new Map(); + +function updateJob(id, update) { + const job = jobs.get(id); + if (job) jobs.set(id, { ...job, ...update, updatedAt: Date.now() }); +} + +function log(jobId, msg) { + const j = jobs.get(jobId); + if (j) jobs.set(jobId, { ...j, log: [...(j.log || []), msg], message: msg }); + console.log(`[${jobId}] ${msg}`); +} + +module.exports = { jobs, updateJob, log }; diff --git a/backend/lib/metadata.js b/backend/lib/metadata.js new file mode 100644 index 0000000..cb69877 --- /dev/null +++ b/backend/lib/metadata.js @@ -0,0 +1,143 @@ +const https = require('https'); +const path = require('path'); +const fs = require('fs'); + +function sanitize(s) { + return (s || 'Unknown').replace(/[\\/:*?"<>|]/g, '_').trim().slice(0, 100); +} + +function navidromeArtist(name) { + if (!name) return name; + return name + .replace(/\s+feat\.?\s+/gi, ' / ') + .replace(/\s+ft\.?\s+/gi, ' / ') + .replace(/\s+featuring\s+/gi, ' / ') + .replace(/,\s+/g, ' / '); +} + +// Download hi-res iTunes artwork (1000x1000) to cover.jpg, return path or null +async function downloadArtwork(url, dir) { + if (!url) return null; + const dest = path.join(dir, 'cover.jpg'); + const hiRes = url.replace(/\d+x\d+bb\./, '1000x1000bb.'); + return new Promise(resolve => { + const tryUrl = (u, fallback) => { + const file = fs.createWriteStream(dest); + https.get(u, res => { + if (res.statusCode !== 200) { + res.resume(); + file.close(); + try { fs.unlinkSync(dest); } catch {} + if (fallback) tryUrl(fallback, null); + else resolve(null); + return; + } + res.pipe(file); + file.on('finish', () => resolve(dest)); + file.on('error', () => resolve(null)); + }).on('error', () => { + if (fallback) tryUrl(fallback, null); + else resolve(null); + }); + }; + tryUrl(hiRes, url !== hiRes ? url : null); + }); +} + +const AUDIO_EXTS = ['mp3', 'flac', 'opus', 'm4a', 'ogg', 'wav']; + +function findExistingTrack(dir, trackPrefix, safeTitle) { + return AUDIO_EXTS + .map(ext => path.join(dir, `${trackPrefix}${safeTitle}.${ext}`)) + .find(f => fs.existsSync(f)) || null; +} + +// Fetch plain + synced lyrics from LRCLIB +async function fetchLyrics(artist, title, album) { + try { + const p = new URLSearchParams({ artist_name: artist, track_name: title }); + if (album) p.set('album_name', album); + const r = await fetch(`https://lrclib.net/api/get?${p}`, { + headers: { 'User-Agent': 'NovaSonic/1.0' }, + signal: AbortSignal.timeout(8000), + }); + if (!r.ok) return null; + const d = await r.json(); + return { plain: d.plainLyrics || null, synced: d.syncedLyrics || null }; + } catch { return null; } +} + +// iTunes track lookup — fills in album, year, trackNum, totalTracks, genre when missing +async function lookupItunesTrackMeta(artist, title) { + try { + const url = `https://itunes.apple.com/search?term=${encodeURIComponent(artist + ' ' + title)}&media=music&entity=musicTrack&limit=5&country=us&explicit=Yes`; + const results = await new Promise((resolve, reject) => { + https.get(url, res => { + let d = ''; res.on('data', c => d += c); + res.on('end', () => { try { resolve(JSON.parse(d).results || []); } catch { reject(new Error('parse')); } }); + }).on('error', reject); + }); + const norm = s => s?.toLowerCase().trim() || ''; + const match = results.find(t => + norm(t.trackName) === norm(title) && + norm(t.artistName).split(' ')[0] === norm(artist).split(' ')[0] + ); + if (!match) return {}; + return { + album: match.collectionName || '', + year: match.releaseDate ? new Date(match.releaseDate).getFullYear() : undefined, + trackNum: match.trackNumber || undefined, + totalTracks: match.trackCount || undefined, + discNum: match.discNumber || undefined, + discTotal: match.discCount || undefined, + genre: match.primaryGenreName || undefined, + artworkUrl: (match.artworkUrl100 || '').replace('100x100bb', '1000x1000bb'), + durationMs: match.trackTimeMillis || undefined, + }; + } catch { return {}; } +} + +// Fetch individual tracks for an iTunes collectionId +async function fetchItunesTracks(collectionId) { + return new Promise((resolve, reject) => { + const url = `https://itunes.apple.com/lookup?id=${collectionId}&entity=song&explicit=Yes`; + https.get(url, res => { + let data = ''; + res.on('data', d => data += d); + res.on('end', () => { + try { + const parsed = JSON.parse(data); + resolve((parsed.results || []).filter(r => r.wrapperType === 'track')); + } catch { reject(new Error('iTunes API parse error')); } + }); + }).on('error', reject); + }); +} + +// Search iTunes for album tracks by artist + album name (fallback when no collectionId) +async function searchItunesAlbumTracks(artist, album) { + return new Promise(resolve => { + const q = encodeURIComponent(`${artist} ${album}`); + https.get(`https://itunes.apple.com/search?term=${q}&media=music&entity=album&limit=5&country=us&explicit=Yes`, res => { + let d = ''; + res.on('data', c => d += c); + res.on('end', () => { + try { + const results = JSON.parse(d).results || []; + const match = results.find(r => + r.collectionName?.toLowerCase().includes(album.toLowerCase()) || + r.artistName?.toLowerCase().includes(artist.toLowerCase()) + ); + if (match) fetchItunesTracks(match.collectionId).then(resolve).catch(() => resolve([])); + else resolve([]); + } catch { resolve([]); } + }); + }).on('error', () => resolve([])); + }); +} + +module.exports = { + sanitize, navidromeArtist, + downloadArtwork, AUDIO_EXTS, findExistingTrack, + fetchLyrics, lookupItunesTrackMeta, fetchItunesTracks, searchItunesAlbumTracks, +}; diff --git a/backend/lib/squid.js b/backend/lib/squid.js new file mode 100644 index 0000000..e55c927 --- /dev/null +++ b/backend/lib/squid.js @@ -0,0 +1,291 @@ +const crypto = require('crypto'); +const https = require('https'); +const http = require('http'); +const path = require('path'); +const fs = require('fs'); +const { spawn } = require('child_process'); +const CONFIG = require('./config'); +const { log } = require('./jobs'); +const { + sanitize, navidromeArtist, + downloadArtwork, fetchLyrics, findExistingTrack, +} = require('./metadata'); +const { embedMetadata } = require('./ytdlp'); + +const SQUID_BASE = 'https://qobuz.squid.wtf'; + +// Qobuz quality codes: mp3=5 (320kbps), flac=6 (16/44), hi-res=7 (24/96) +const QUALITY_CODE = { mp3: '5', flac: '6', 'hi-res': '7' }; + +function squidExt() { + return CONFIG.squidQuality === 'mp3' ? 'mp3' : 'flac'; +} + +// ─── ALTCHA cookie cache ───────────────────────────────────────────────────── +let _captchaCookie = null; +let _captchaAt = 0; +const CAPTCHA_TTL = 27 * 60 * 1000; // refresh 1 min before server's 28-min expiry + +// Iterative SHA-256 PoW solver — yields every 1000 iterations to stay non-blocking +async function solveAltcha(challenge) { + const { nonce, salt, cost, keyLength, keyPrefix } = challenge; + const saltBuf = Buffer.from(salt, 'hex'); + const nonceBuf = Buffer.from(nonce, 'hex'); + const prefixBuf = Buffer.from(keyPrefix, 'hex'); + const startMs = Date.now(); + const BATCH = 1000; + + for (let base = 0; base < 1_000_000; base += BATCH) { + for (let i = 0; i < BATCH; i++) { + const counter = base + i; + const ctrBuf = Buffer.allocUnsafe(4); + ctrBuf.writeUInt32BE(counter); + // password = nonce || counter_be32 ; initial = salt || password + const initial = Buffer.concat([saltBuf, nonceBuf, ctrBuf]); + let derived = crypto.createHash('sha256').update(initial).digest().slice(0, keyLength); + for (let c = 0; c < cost; c++) { + derived = crypto.createHash('sha256').update(derived).digest().slice(0, keyLength); + } + if (derived.slice(0, prefixBuf.length).equals(prefixBuf)) { + return { counter, derivedKey: derived.toString('hex'), time: Date.now() - startMs }; + } + } + // Yield to event loop between batches + await new Promise(r => setImmediate(r)); + } + throw new Error('ALTCHA: no solution found within 1M iterations'); +} + +function httpsGet(url) { + return new Promise((resolve, reject) => { + const mod = url.startsWith('https') ? https : http; + mod.get(url, { headers: { 'User-Agent': 'NovaSonic/1.0' } }, res => { + let d = ''; + res.on('data', c => d += c); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: d })); + }).on('error', reject); + }); +} + +function httpsPost(url, body, extraHeaders) { + return new Promise((resolve, reject) => { + const payload = JSON.stringify(body); + const parsed = new URL(url); + const opts = { + hostname: parsed.hostname, + port: parsed.port || 443, + path: parsed.pathname + parsed.search, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(payload), + 'User-Agent': 'NovaSonic/1.0', + ...extraHeaders, + }, + }; + const req = https.request(opts, res => { + let d = ''; + res.on('data', c => d += c); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: d })); + }); + req.on('error', reject); + req.write(payload); + req.end(); + }); +} + +async function getAltchaCookie() { + if (_captchaCookie && Date.now() - _captchaAt < CAPTCHA_TTL) return _captchaCookie; + + // 1. Fetch challenge + const chRes = await httpsGet(`${SQUID_BASE}/api/altcha/challenge`); + if (chRes.status !== 200) throw new Error(`ALTCHA challenge HTTP ${chRes.status}`); + const challenge = JSON.parse(chRes.body); + + // 2. Solve PoW + const solution = await solveAltcha(challenge); + + // 3. Verify — server sets captcha_verified_at cookie + const payload = Buffer.from(JSON.stringify({ challenge, solution })).toString('base64'); + const vRes = await httpsPost(`${SQUID_BASE}/api/altcha/verify`, { payload }); + if (vRes.status !== 200) throw new Error(`ALTCHA verify HTTP ${vRes.status}`); + + // Parse Set-Cookie: captcha_verified_at=; ... + const setCookie = [vRes.headers['set-cookie']].flat().filter(Boolean); + const cookiePair = setCookie.map(c => c.split(';')[0]).find(c => c.startsWith('captcha_verified_at=')); + if (!cookiePair) throw new Error('ALTCHA verify: no cookie in response'); + + _captchaCookie = cookiePair; + _captchaAt = Date.now(); + return _captchaCookie; +} + +async function squidApiGet(endpoint) { + const cookie = await getAltchaCookie(); + return new Promise((resolve, reject) => { + const url = `${SQUID_BASE}${endpoint}`; + const parsed = new URL(url); + const opts = { + hostname: parsed.hostname, + port: parsed.port || 443, + path: parsed.pathname + parsed.search, + method: 'GET', + headers: { + 'Cookie': cookie, + 'Token-Country': 'US', + 'User-Agent': 'NovaSonic/1.0', + }, + }; + const req = https.request(opts, res => { + let d = ''; + res.on('data', c => d += c); + res.on('end', () => resolve({ status: res.statusCode, body: d })); + }); + req.on('error', reject); + req.end(); + }); +} + +// Search Qobuz via squid.wtf, return best matching track object or null +async function squidSearch(artist, title, durationMs) { + const query = encodeURIComponent(`${artist} ${title}`); + const res = await squidApiGet(`/api/get-music?q=${query}&offset=0`); + if (res.status !== 200) throw new Error(`squid search HTTP ${res.status}`); + + const json = JSON.parse(res.body); + // Handle both {data:{tracks:{items:[...]}}} and {tracks:{items:[...]}} wrappers + const items = json?.data?.tracks?.items ?? json?.tracks?.items ?? []; + if (!items.length) return null; + + const norm = s => (s || '').toLowerCase().trim(); + const normT = norm(title); + const normA = norm(artist).split(/[,\s]/)[0]; + + // Score candidates: prefer exact title match + matching artist + duration proximity + const scored = items.map(t => { + const tTitle = norm(t.title); + const tArtist = norm(t.performer?.name || ''); + const tDuration = (t.duration || 0) * 1000; // Qobuz uses seconds + + let score = 0; + if (tTitle === normT) score += 100; + else if (tTitle.includes(normT)) score += 50; + if (tArtist.startsWith(normA)) score += 30; + if (durationMs && tDuration) { + const diff = Math.abs(tDuration - durationMs); + if (diff < 3000) score += 20; // within 3 s + else if (diff < 10000) score += 10; // within 10 s + else if (diff > 30000) score -= 20; // way off + } + return { track: t, score }; + }); + + scored.sort((a, b) => b.score - a.score); + const best = scored[0]; + // Reject if score is too low (no reasonable match) + if (best.score < 30) return null; + return best.track; +} + +// Stream a URL to destPath (handles redirects via Location header) +async function downloadUrlToFile(url, destPath) { + return new Promise((resolve, reject) => { + const tryFetch = (u, hops) => { + if (hops > 5) { reject(new Error('Too many redirects')); return; } + const mod = u.startsWith('https') ? https : http; + const parsed = new URL(u); + const opts = { + hostname: parsed.hostname, + port: parsed.port || (u.startsWith('https') ? 443 : 80), + path: parsed.pathname + parsed.search, + method: 'GET', + headers: { 'User-Agent': 'NovaSonic/1.0' }, + }; + const req = mod.request(opts, res => { + if (res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 307) { + res.resume(); + tryFetch(res.headers.location, hops + 1); + return; + } + if (res.statusCode !== 200) { + res.resume(); + reject(new Error(`Download HTTP ${res.statusCode}`)); + return; + } + const file = fs.createWriteStream(destPath); + res.pipe(file); + file.on('finish', () => file.close(resolve)); + file.on('error', reject); + }); + req.on('error', reject); + req.end(); + }; + tryFetch(url, 0); + }); +} + +// Download a single track via squid.wtf / Qobuz. +// Returns { success: true, dir, skipped? } on success, null if track not found on Qobuz. +async function downloadTrack({ jobId, artist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs }) { + const mainArtist = albumArtist || artist; + const safeArtist = sanitize(mainArtist.split(',')[0].trim()); + const safeAlbum = sanitize(album || 'Singles'); + const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum); + fs.mkdirSync(dir, { recursive: true }); + + const safeTitle = sanitize(title); + const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : ''; + const ext = squidExt(); + + const existing = findExistingTrack(dir, trackPrefix, safeTitle); + if (existing) { + log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`); + return { success: true, dir, skipped: true }; + } + + log(jobId, `🦑 squid.wtf: suche "${artist} – ${title}"`); + + // Solve ALTCHA if needed (cached for 27 min) + await getAltchaCookie(); + + // Find track on Qobuz + const searchArtist = mainArtist.replace(/[^\p{L}\p{N}\s,&\-]/gu, '').trim(); + const track = await squidSearch(searchArtist, title, durationMs); + if (!track) { + log(jobId, `⚠️ squid.wtf: "${title}" nicht auf Qobuz gefunden`); + return null; // signals caller to fall back to yt-dlp + } + + // Get download URL + const quality = QUALITY_CODE[CONFIG.squidQuality] || '5'; + const dlRes = await squidApiGet(`/api/download-music?track_id=${track.id}&quality=${quality}`); + if (dlRes.status !== 200) throw new Error(`squid download-music HTTP ${dlRes.status}`); + + const dlJson = JSON.parse(dlRes.body); + const streamUrl = dlJson?.data?.url ?? dlJson?.url; + if (!streamUrl) throw new Error('squid: no download URL in response'); + + // Download to a temp file, then embed metadata + const audioFile = path.join(dir, `${trackPrefix}${safeTitle}.${ext}`); + const tmpFile = audioFile + '.squidtmp'; + log(jobId, `⬇️ Lade herunter (Qobuz ${CONFIG.squidQuality.toUpperCase()})…`); + try { + await downloadUrlToFile(streamUrl, tmpFile); + fs.renameSync(tmpFile, audioFile); + } catch (e) { + try { fs.unlinkSync(tmpFile); } catch {} + throw e; + } + + // Use Qobuz artwork if no better URL was passed in + const finalArtwork = artworkUrl || track.album?.image?.large || track.album?.image?.small || ''; + + return embedMetadata({ + jobId, audioFile, dir, trackPrefix, safeTitle, + artist, mainArtist, album, year, trackNum, totalTracks, + discNum, discTotal, compilation, genre, + artworkUrl: finalArtwork, explicit, title, + }); +} + +module.exports = { downloadTrack }; diff --git a/backend/lib/ytdlp.js b/backend/lib/ytdlp.js new file mode 100644 index 0000000..5fd5823 --- /dev/null +++ b/backend/lib/ytdlp.js @@ -0,0 +1,276 @@ +const { spawn } = require('child_process'); +const path = require('path'); +const fs = require('fs'); +const CONFIG = require('./config'); +const { log } = require('./jobs'); +const { + sanitize, navidromeArtist, + downloadArtwork, fetchLyrics, findExistingTrack, +} = require('./metadata'); + +// Extract a plain word from an artist name for yt-dlp --match-filter uploader check. +function artistFilterWord(name) { + const first = (name || '').split(/[,/•]/)[0].trim(); + const runs = first.match(/\w+/g) || []; + const best = runs.filter(r => r.length >= 3).sort((a, b) => b.length - a.length)[0]; + return best || ''; +} + +// Extract distinctive words from a track title for yt-dlp title ~= (?i) filters. +function buildTitleKeywords(title) { + const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']); + const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']); + const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit']; + const lower = (title || '').toLowerCase(); + const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim(); + const words = strippedTitle + .replace(/[^\w\s]/g, ' ') + .split(/\s+/) + .filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase())); + const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2); + const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w; + const result = top2.map(toPattern); + const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null; + if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier); + return result; +} + +// Build yt-dlp --match-filter conditions to reject variant versions. +function buildVariantFilter(title) { + const t = (title || '').toLowerCase(); + const RULES = [ + { key: 'sped', expr: 'title !*= sped' }, + { key: 'speedup', expr: 'title !*= speedup' }, + { key: 'slowed', expr: 'title !*= slowed' }, + { key: 'reverb', expr: 'title !*= reverb' }, + { key: 'nightcore', expr: 'title !*= nightcore' }, + { key: 'karaoke', expr: 'title !*= karaoke' }, + { key: 'parody', expr: 'title !*= parody' }, + { key: 'remix', expr: 'title !*= remix' }, + { key: 'cover', expr: 'title !*= cover' }, + { key: 'acoustic', expr: 'title !*= acoustic' }, + { key: 'instrumental', expr: 'title !*= instrumental' }, + ]; + const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr); + return parts.length ? parts.join(' & ') : ''; +} + +async function downloadTrack({ jobId, artist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs }) { + const mainArtist = albumArtist || artist; + const safeArtist = sanitize(mainArtist.split(',')[0].trim()); + const safeAlbum = sanitize(album || 'Singles'); + const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum); + fs.mkdirSync(dir, { recursive: true }); + + const safeTitle = sanitize(title); + const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : ''; + + const existing = findExistingTrack(dir, trackPrefix, safeTitle); + if (existing) { + log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`); + return { success: true, dir, skipped: true }; + } + + const outputTemplate = path.join(dir, `${trackPrefix}${safeTitle}.%(ext)s`); + const searchArtist = mainArtist.replace(/[^\p{L}\p{N}\s,&\-]/gu, '').trim(); + const query = `${searchArtist} - ${title}`; + + const baseArgs = [ + '--extract-audio', + '--audio-format', CONFIG.audioFmt, + '--audio-quality', CONFIG.audioQuality, + '-o', outputTemplate, + '--no-playlist', + '--no-warnings', + '--newline', + '--match-filter', 'webpage_url !*= /browse/', + ]; + const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1']; + const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1']; + const PI_UNFILTERED = ['--playlist-items', '1']; + + const spawnYtdlp = (args) => new Promise((resolve, reject) => { + const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } }); + let stderr = '', lastPct = ''; + proc.stdout.on('data', d => { + d.toString().split('\n').filter(l => l.trim()).forEach(line => { + if (line.includes('%') && line === lastPct) return; + if (line.includes('%')) lastPct = line; + log(jobId, line.trim()); + }); + }); + proc.stderr.on('data', d => { + stderr += d.toString(); + d.toString().split('\n') + .filter(l => l.trim() && !l.startsWith('[debug]')) + .forEach(l => log(jobId, l.trim())); + }); + proc.on('error', reject); + proc.on('close', code => + (code === 0 || code === 101) ? resolve() : reject(new Error(`yt-dlp code ${code}: ${stderr.slice(-300)}`)) + ); + }); + + const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D'; + const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(query)}¶ms=${YTM_SONGS_PARAM}`; + const expectedAudio = path.join(dir, `${trackPrefix}${safeTitle}.${CONFIG.audioFmt}`); + const hasAudio = () => fs.existsSync(expectedAudio); + const tryYtdlp = async (args) => { await spawnYtdlp(args); }; + + const variantFilter = buildVariantFilter(title); + const titleKws = buildTitleKeywords(title); + const variantPart = variantFilter ? ` & ${variantFilter}` : ''; + const titleKwPart = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : ''; + + const mf = (dur, variant) => { + const durPart = dur ? `duration > ${dur.min} & duration < ${dur.max} & ` : ''; + return ['--match-filter', + `${durPart}!is_live${titleKwPart}${variant ? variantPart : ''}`]; + }; + + if (durationMs) { + const durationSec = Math.round(durationMs / 1000); + const margin = Math.max(15, Math.round(durationSec * 0.15)); + const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin }; + + log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`); + await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]); + if (!hasAudio()) { + log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…'); + await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…'); + await tryYtdlp([ytmUrl, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…'); + await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Letzter gefilterter Versuch…'); + await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Letzter Versuch: keine Filter…'); + await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]); + } + } else { + log(jobId, `🔍 Suche YT Music: "${query}"`); + await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); + if (!hasAudio()) { + await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Letzter gefilterter Versuch…'); + await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); + } + if (!hasAudio()) { + log(jobId, '⚠️ Letzter Versuch: keine Filter…'); + await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]); + } + if (!hasAudio()) { + await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]); + } + } + + const audioFile = hasAudio() ? expectedAudio : null; + if (!audioFile) { + log(jobId, '⚠️ Keine Audio-Datei gefunden'); + return { success: true, dir }; + } + + return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title, lyricsArtist: artist }); +} + +// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths +async function embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title }) { + const [coverPath, lyricsData] = await Promise.all([ + downloadArtwork(artworkUrl, dir), + fetchLyrics(artist, title, album), + ]); + const hasCover = !!(coverPath && fs.existsSync(coverPath)); + + if (lyricsData?.synced) { + const lrcPath = path.join(dir, `${trackPrefix}${safeTitle}.lrc`); + fs.writeFileSync(lrcPath, lyricsData.synced, 'utf8'); + log(jobId, '🎵 Synced lyrics gefunden'); + } else if (lyricsData?.plain) { + log(jobId, '📝 Lyrics gefunden'); + } + + log(jobId, `🏷 Metadaten + Cover → ${path.basename(audioFile)}`); + + const fileExt = path.extname(audioFile); + const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt; + await new Promise(resolve => { + const ffArgs = ['-i', audioFile]; + if (hasCover) ffArgs.push('-i', coverPath); + ffArgs.push('-map', '0:a'); + if (hasCover) ffArgs.push('-map', '1:v'); + ffArgs.push( + '-c', 'copy', + '-map_metadata', '-1', + '-id3v2_version', '3', + '-metadata', `title=${title}`, + '-metadata', `artist=${navidromeArtist(artist)}`, + '-metadata', `album_artist=${mainArtist}`, + ); + if (album) ffArgs.push('-metadata', `album=${album}`); + if (year) ffArgs.push('-metadata', `date=${String(year)}`); + if (trackNum) ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`); + if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`); + if (genre) ffArgs.push('-metadata', `genre=${genre}`); + if (discNum) ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`); + if (compilation) ffArgs.push('-metadata', `compilation=1`); + if (explicit) ffArgs.push('-metadata', 'ITUNESADVISORY=1'); + if (lyricsData?.plain) ffArgs.push('-metadata', `lyrics=${lyricsData.plain}`); + if (hasCover) { + ffArgs.push( + '-metadata:s:v', 'title=Album cover', + '-metadata:s:v', 'comment=Cover (front)', + ); + } + ffArgs.push(tmpFile, '-y'); + + const ff = spawn('ffmpeg', ffArgs); + let ffErr = ''; + ff.stderr.on('data', d => ffErr += d.toString()); + ff.on('error', () => resolve()); + ff.on('close', code => { + if (code === 0 && fs.existsSync(tmpFile)) { + fs.renameSync(tmpFile, audioFile); + if (hasCover) try { fs.unlinkSync(coverPath); } catch {} + const base = path.basename(audioFile, path.extname(audioFile)); + for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) { + try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {} + } + log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`); + } else { + try { fs.unlinkSync(tmpFile); } catch {} + log(jobId, `⚠️ ffmpeg code ${code}`); + if (ffErr) log(jobId, ffErr.slice(-200)); + } + resolve(); + }); + }); + + return { success: true, dir }; +} + +module.exports = { downloadTrack, embedMetadata }; diff --git a/backend/novasonic-server.js b/backend/novasonic-server.js index ca7f8c7..4781d23 100644 --- a/backend/novasonic-server.js +++ b/backend/novasonic-server.js @@ -1,6 +1,6 @@ /** * NovaSonic Backend Server - * Downloads music via yt-dlp and saves directly into your Navidrome music library + * Downloads music via yt-dlp / squid.wtf (Qobuz) and saves into your Navidrome library. * * Requirements: * - Node.js 18+ @@ -8,21 +8,16 @@ * - ffmpeg installed */ -const express = require('express'); -const { spawn, execSync } = require('child_process'); -const path = require('path'); -const fs = require('fs'); -const https = require('https'); +const express = require('express'); +const { execSync } = require('child_process'); +const path = require('path'); +const fs = require('fs'); +const https = require('https'); -// ─── CONFIG ─────────────────────────────────────────────────────────────────── -const CONFIG = { - port: process.env.PORT || 3700, - musicPath: process.env.MUSIC_PATH || '/music', - ytdlpBin: process.env.YTDLP_BIN || 'yt-dlp', - audioFmt: process.env.AUDIO_FMT || 'mp3', - audioQuality: process.env.AUDIO_QUALITY || '0', - maxParallel: parseInt(process.env.MAX_PARALLEL || '2'), -}; +const CONFIG = require('./lib/config'); +const { jobs, updateJob, log } = require('./lib/jobs'); +const { lookupItunesTrackMeta, fetchItunesTracks, searchItunesAlbumTracks } = require('./lib/metadata'); +const { downloadTrack, downloadAlbumTracks } = require('./lib/downloader'); // ─── INIT ───────────────────────────────────────────────────────────────────── const app = express(); @@ -36,8 +31,6 @@ app.use((req, res, next) => { }); app.use(express.json()); -const jobs = new Map(); - let ytdlpVersion = 'unknown'; try { ytdlpVersion = execSync(`${CONFIG.ytdlpBin} --version`, { timeout: 5000 }).toString().trim(); @@ -51,502 +44,6 @@ if (!fs.existsSync(CONFIG.musicPath)) { fs.mkdirSync(CONFIG.musicPath, { recursive: true }); } -// ─── HELPERS ────────────────────────────────────────────────────────────────── -function sanitize(s) { - return (s || 'Unknown').replace(/[\\/:*?"<>|]/g, '_').trim().slice(0, 100); -} - -// Extract a plain word from an artist name for yt-dlp --match-filter uploader check. -// Uses longest contiguous \w+ run so special chars like "N!GHT" → "N" + "GHT" picks "GHT", -// but actually picks longest: "N!GHT" → runs ["N","GHT"] → "GHT" (3 chars). -// "☆NoAki☆" → "NoAki", "Chiru-san" → "Chiru", "A, B" → "A" -function artistFilterWord(name) { - const first = (name || '').split(/[,/•]/)[0].trim(); - const runs = first.match(/\w+/g) || []; - const best = runs.filter(r => r.length >= 3).sort((a, b) => b.length - a.length)[0]; - return best || ''; -} - -// Extract distinctive words from a track title for case-insensitive yt-dlp title ~= (?i) filters. -// Returns up to 2 of the most distinctive words so the filter is precise: -// "Echoes of Eternity" → ["Eternity", "Echoes"] → title ~= (?i)Eternity & title ~= (?i)Echoes -// "Amethyst Flowers" → ["Amethyst", "Flowers?"] → title ~= (?i)Amethyst & title ~= (?i)Flowers? -// "Whispers" → ["Whispers?"] → title ~= (?i)Whispers? -// "Eclipse - Radio Mix"→ ["Eclipse", "Radio"] → title ~= (?i)Eclipse & title ~= (?i)Radio -// Trailing 's' is made optional (Whispers? matches "Whisper" or "Whispers") to handle -// cases where the YouTube video title drops the plural form. -function buildTitleKeywords(title) { - const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']); - const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']); - const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit']; - const lower = (title || '').toLowerCase(); - // Strip parenthetical genre/style tags like "(Jersey Club)", "(feat. X)" before picking keywords - // so "Eleven Night (Jersey Club)" → keywords from "Eleven Night", not "Jersey" - const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim(); - const words = strippedTitle - .replace(/[^\w\s]/g, ' ') - .split(/\s+/) - .filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase())); - // Take top 2 by length — two words together are far more precise than one alone - const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2); - // Make trailing 's' optional so "Whispers?" matches "Whisper" or "Whispers" - const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w; - const result = top2.map(toPattern); - // Add mix qualifier only if neither top word already covers it - const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null; - if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier); - return result; -} - -// Build yt-dlp --match-filter conditions to reject variant versions (speed up, slowed, remix…) -// unless the original track title already contains those keywords. -function buildVariantFilter(title) { - const t = (title || '').toLowerCase(); - const RULES = [ - { key: 'sped', expr: 'title !*= sped' }, // "sped up" - { key: 'speedup', expr: 'title !*= speedup' }, // "speedup" - { key: 'slowed', expr: 'title !*= slowed' }, - { key: 'reverb', expr: 'title !*= reverb' }, - { key: 'nightcore', expr: 'title !*= nightcore' }, - { key: 'karaoke', expr: 'title !*= karaoke' }, - { key: 'parody', expr: 'title !*= parody' }, - { key: 'remix', expr: 'title !*= remix' }, - { key: 'cover', expr: 'title !*= cover' }, - { key: 'acoustic', expr: 'title !*= acoustic' }, - { key: 'instrumental', expr: 'title !*= instrumental' }, - ]; - const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr); - return parts.length ? parts.join(' & ') : ''; -} - -// Convert multi-artist strings to " / " separator so Navidrome splits correctly. -// Handles: "feat." / "ft." / "featuring" and comma-separated artists from Spotify/iTunes. -function navidromeArtist(name) { - if (!name) return name; - return name - .replace(/\s+feat\.?\s+/gi, ' / ') - .replace(/\s+ft\.?\s+/gi, ' / ') - .replace(/\s+featuring\s+/gi, ' / ') - .replace(/,\s+/g, ' / '); // "Artist A, Artist B" → "Artist A / Artist B" -} - -function updateJob(id, update) { - const job = jobs.get(id); - if (job) jobs.set(id, { ...job, ...update, updatedAt: Date.now() }); -} - -function log(jobId, msg) { - const j = jobs.get(jobId); - if (j) jobs.set(jobId, { ...j, log: [...(j.log || []), msg], message: msg }); - console.log(`[${jobId}] ${msg}`); -} - -// Download hi-res iTunes artwork (1000x1000) to cover.jpg, return path or null -async function downloadArtwork(url, dir) { - if (!url) return null; - const dest = path.join(dir, 'cover.jpg'); - // Upgrade to 1000x1000 — iTunes URL pattern: ...100x100bb.jpg - const hiRes = url.replace(/\d+x\d+bb\./, '1000x1000bb.'); - return new Promise(resolve => { - const tryUrl = (u, fallback) => { - const file = fs.createWriteStream(dest); - https.get(u, res => { - if (res.statusCode !== 200) { - res.resume(); - file.close(); - try { fs.unlinkSync(dest); } catch {} - if (fallback) tryUrl(fallback, null); - else resolve(null); - return; - } - res.pipe(file); - file.on('finish', () => resolve(dest)); - file.on('error', () => resolve(null)); - }).on('error', () => { - if (fallback) tryUrl(fallback, null); - else resolve(null); - }); - }; - tryUrl(hiRes, url !== hiRes ? url : null); - }); -} - -const AUDIO_EXTS = ['mp3', 'flac', 'opus', 'm4a', 'ogg', 'wav']; - -function findExistingTrack(dir, trackPrefix, safeTitle) { - return AUDIO_EXTS - .map(ext => path.join(dir, `${trackPrefix}${safeTitle}.${ext}`)) - .find(f => fs.existsSync(f)) || null; -} - -// Fetch plain + synced lyrics from LRCLIB -async function fetchLyrics(artist, title, album) { - try { - const p = new URLSearchParams({ artist_name: artist, track_name: title }); - if (album) p.set('album_name', album); - const r = await fetch(`https://lrclib.net/api/get?${p}`, { - headers: { 'User-Agent': 'NovaSonic/1.0' }, - signal: AbortSignal.timeout(8000), - }); - if (!r.ok) return null; - const d = await r.json(); - return { plain: d.plainLyrics || null, synced: d.syncedLyrics || null }; - } catch { return null; } -} - -// iTunes track lookup — fills in album, year, trackNum, totalTracks, genre when missing -async function lookupItunesTrackMeta(artist, title) { - try { - const url = `https://itunes.apple.com/search?term=${encodeURIComponent(artist + ' ' + title)}&media=music&entity=musicTrack&limit=5&country=us&explicit=Yes`; - const results = await new Promise((resolve, reject) => { - https.get(url, res => { - let d = ''; res.on('data', c => d += c); - res.on('end', () => { try { resolve(JSON.parse(d).results || []); } catch { reject(new Error('parse')); } }); - }).on('error', reject); - }); - const norm = s => s?.toLowerCase().trim() || ''; - const match = results.find(t => - norm(t.trackName) === norm(title) && - norm(t.artistName).split(' ')[0] === norm(artist).split(' ')[0] - ); - if (!match) return {}; - return { - album: match.collectionName || '', - year: match.releaseDate ? new Date(match.releaseDate).getFullYear() : undefined, - trackNum: match.trackNumber || undefined, - totalTracks: match.trackCount || undefined, - discNum: match.discNumber || undefined, - discTotal: match.discCount || undefined, - genre: match.primaryGenreName || undefined, - artworkUrl: (match.artworkUrl100 || '').replace('100x100bb', '1000x1000bb'), - durationMs: match.trackTimeMillis || undefined, - }; - } catch { return {}; } -} - -// Download a single track: yt-dlp audio + ffmpeg clean metadata + iTunes cover embedded -async function downloadTrack({ jobId, artist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs }) { - const mainArtist = albumArtist || artist; - const safeArtist = sanitize(mainArtist.split(',')[0].trim()); - const safeAlbum = sanitize(album || 'Singles'); - const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum); - fs.mkdirSync(dir, { recursive: true }); - - const safeTitle = sanitize(title); - const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : ''; - - const existing = findExistingTrack(dir, trackPrefix, safeTitle); - if (existing) { - log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`); - return { success: true, dir, skipped: true }; - } - - const outputTemplate = path.join(dir, `${trackPrefix}${safeTitle}.%(ext)s`); - // Strip symbol chars (e.g. ☆) but keep Unicode letters like ø, ä, é - const searchArtist = mainArtist.replace(/[^\p{L}\p{N}\s,&\-]/gu, '').trim(); - const query = `${searchArtist} - ${title}`; - - // Step 1: yt-dlp — audio only, no thumbnail (we embed iTunes artwork ourselves) - const baseArgs = [ - '--extract-audio', - '--audio-format', CONFIG.audioFmt, - '--audio-quality', CONFIG.audioQuality, - '-o', outputTemplate, - '--no-playlist', - '--no-warnings', - '--newline', - // Skip artist channel pages that YTM sometimes returns as search result 1; - // without this yt-dlp recursively fetches the entire channel uploads playlist. - '--match-filter', 'webpage_url !*= /browse/', - ]; - // Filtered searches: top 5 (fast) and top 10 (wider net when top 5 misses). - // Unfiltered fallback only needs item 1 (no filter to skip past). - const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1']; - const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1']; - const PI_UNFILTERED = ['--playlist-items', '1']; - - const spawnYtdlp = (args) => new Promise((resolve, reject) => { - const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } }); - let stderr = '', lastPct = ''; - proc.stdout.on('data', d => { - d.toString().split('\n').filter(l => l.trim()).forEach(line => { - if (line.includes('%') && line === lastPct) return; - if (line.includes('%')) lastPct = line; - log(jobId, line.trim()); - }); - }); - proc.stderr.on('data', d => { - stderr += d.toString(); - d.toString().split('\n') - .filter(l => l.trim() && !l.startsWith('[debug]')) - .forEach(l => log(jobId, l.trim())); - }); - proc.on('error', reject); - proc.on('close', code => - // 101 = --max-downloads reached (not an error — file was downloaded successfully) - (code === 0 || code === 101) ? resolve() : reject(new Error(`yt-dlp code ${code}: ${stderr.slice(-300)}`)) - ); - }); - - // ¶ms= is YTM's "Songs only" filter (same as clicking "Titel" tab in the browser). - // Without it yt-dlp gets a mixed feed of videos, channels, and playlists as well. - const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D'; - const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(query)}¶ms=${YTM_SONGS_PARAM}`; - // We know the exact output filename before yt-dlp runs — no snapshot heuristic needed. - const expectedAudio = path.join(dir, `${trackPrefix}${safeTitle}.${CONFIG.audioFmt}`); - const hasAudio = () => fs.existsSync(expectedAudio); - const tryYtdlp = async (args) => { await spawnYtdlp(args); }; - - const variantFilter = buildVariantFilter(title); - const titleKws = buildTitleKeywords(title); - const variantPart = variantFilter ? ` & ${variantFilter}` : ''; - // ~= uses Python re.search; (?i) inline flag = case-insensitive substring match - const titleKwPart = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : ''; - - // Build a --match-filter arg. Artist filter removed — `artist` field is unreliable in yt-dlp - // for YTM (often empty for label-distributed tracks). Search query already targets the right - // artist; title keyword + duration are sufficient guards against wrong results. - const mf = (dur, variant) => { - const durPart = dur ? `duration > ${dur.min} & duration < ${dur.max} & ` : ''; - return ['--match-filter', - `${durPart}!is_live${titleKwPart}${variant ? variantPart : ''}`]; - }; - - if (durationMs) { - const durationSec = Math.round(durationMs / 1000); - const margin = Math.max(15, Math.round(durationSec * 0.15)); // ±15 % (min 15 s) - const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin }; - - log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`); - // 1. YTM top-5 — Dauer + Titel + Varianten-Filter - await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]); - if (!hasAudio()) { - // 2. YT top-5 — Dauer + Titel + Varianten-Filter - log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…'); - await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]); - } - if (!hasAudio()) { - // 3. YTM top-10 — Dauer + Titel (correcte Version evtl. außerhalb top-5) - log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…'); - await tryYtdlp([ytmUrl, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]); - } - if (!hasAudio()) { - // 4. YT top-10 — Dauer + Titel - await tryYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]); - } - if (!hasAudio()) { - // 5. YTM top-5 — Titel + Varianten (kein Dauer-Check) - log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…'); - await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); - } - if (!hasAudio()) { - // 6. YT top-5 — Titel + Varianten - await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); - } - if (!hasAudio()) { - // 7. YTM top-5 — nur Titel (kein Varianten-Filter) - log(jobId, '⚠️ Letzter gefilterter Versuch…'); - await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); - } - if (!hasAudio()) { - // 8. YT top-5 — nur Titel - await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); - } - if (!hasAudio()) { - // 9+10. Letzter Ausweg: komplett ungefiltert - log(jobId, '⚠️ Letzter Versuch: keine Filter…'); - await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]); - } - if (!hasAudio()) { - await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]); - } - } else { - log(jobId, `🔍 Suche YT Music: "${query}"`); - // 1. YTM — Titel + Varianten - await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); - if (!hasAudio()) { - // 2. YT — Titel + Varianten - await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]); - } - if (!hasAudio()) { - // 3. YTM — nur Titel - log(jobId, '⚠️ Letzter gefilterter Versuch…'); - await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); - } - if (!hasAudio()) { - // 4. YT — nur Titel - await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]); - } - if (!hasAudio()) { - log(jobId, '⚠️ Letzter Versuch: keine Filter…'); - await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]); - } - if (!hasAudio()) { - await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]); - } - } - - // Step 2: locate the audio file yt-dlp wrote (exact expected path) - const audioFile = hasAudio() ? expectedAudio : null; - - if (!audioFile) { - log(jobId, '⚠️ Keine Audio-Datei gefunden'); - return { success: true, dir }; - } - - // Step 3: download artwork + fetch lyrics in parallel - const [coverPath, lyricsData] = await Promise.all([ - downloadArtwork(artworkUrl, dir), - fetchLyrics(artist, title, album), - ]); - const hasCover = !!(coverPath && fs.existsSync(coverPath)); - - if (lyricsData?.synced) { - const lrcPath = path.join(dir, `${trackPrefix}${safeTitle}.lrc`); - fs.writeFileSync(lrcPath, lyricsData.synced, 'utf8'); - log(jobId, '🎵 Synced lyrics gefunden'); - } else if (lyricsData?.plain) { - log(jobId, '📝 Lyrics gefunden'); - } - - log(jobId, `🏷 Metadaten + Cover → ${path.basename(audioFile)}`); - - // Step 4: ffmpeg — strip old tags, write clean ID3, embed hi-res artwork - const fileExt = path.extname(audioFile); - const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt; - await new Promise(resolve => { - const ffArgs = ['-i', audioFile]; - if (hasCover) ffArgs.push('-i', coverPath); - - // Map: audio from input 0, artwork from input 1 (if available) - ffArgs.push('-map', '0:a'); - if (hasCover) ffArgs.push('-map', '1:v'); - - ffArgs.push( - '-c', 'copy', - '-map_metadata', '-1', // strip all existing tags (removes blurry YT thumbnail) - '-id3v2_version', '3', - '-metadata', `title=${title}`, - '-metadata', `artist=${navidromeArtist(artist)}`, - '-metadata', `album_artist=${mainArtist}`, - ); - if (album) ffArgs.push('-metadata', `album=${album}`); - if (year) ffArgs.push('-metadata', `date=${String(year)}`); - if (trackNum) ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`); - if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`); - if (genre) ffArgs.push('-metadata', `genre=${genre}`); - if (discNum) ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`); - if (compilation) ffArgs.push('-metadata', `compilation=1`); - if (explicit) ffArgs.push('-metadata', 'ITUNESADVISORY=1'); - if (lyricsData?.plain) ffArgs.push('-metadata', `lyrics=${lyricsData.plain}`); - if (hasCover) { - ffArgs.push( - '-metadata:s:v', 'title=Album cover', - '-metadata:s:v', 'comment=Cover (front)', - ); - } - ffArgs.push(tmpFile, '-y'); - - const ff = spawn('ffmpeg', ffArgs); - let ffErr = ''; - ff.stderr.on('data', d => ffErr += d.toString()); - ff.on('error', () => resolve()); - ff.on('close', code => { - if (code === 0 && fs.existsSync(tmpFile)) { - fs.renameSync(tmpFile, audioFile); - if (hasCover) try { fs.unlinkSync(coverPath); } catch {} - // clean up any leftover webm the yt-dlp conversion didn't remove - const base = path.basename(audioFile, path.extname(audioFile)); - for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) { - try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {} - } - log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`); - } else { - try { fs.unlinkSync(tmpFile); } catch {} - log(jobId, `⚠️ ffmpeg code ${code}`); - if (ffErr) log(jobId, ffErr.slice(-200)); - } - resolve(); - }); - }); - - return { success: true, dir }; -} - -// Download all tracks of an album sequentially -async function downloadAlbumTracks({ jobId, artist, albumTitle, tracks, artworkUrl }) { - const total = tracks.length; - log(jobId, `📀 Album "${albumTitle}" — ${total} Tracks`); - let done = 0, skipped = 0; - - for (const track of tracks) { - try { - const result = await downloadTrack({ - jobId, - artist: track.artistName || artist, // track artist (may include feat.) - albumArtist: artist, // always the main artist → consistent grouping - title: track.trackName || track.title, - album: albumTitle, - year: track.releaseDate ? new Date(track.releaseDate).getFullYear() : undefined, - trackNum: track.trackNumber || undefined, - totalTracks: total, - discNum: track.discNumber || undefined, - discTotal: track.discCount || undefined, - compilation: track.collectionType === 'Compilation' || undefined, - genre: track.primaryGenreName || undefined, - artworkUrl: artworkUrl || track.artworkUrl100, - explicit: track.trackExplicitness === 'explicit', - outputDir: CONFIG.musicPath, - durationMs: track.trackTimeMillis || undefined, - }); - if (result?.skipped) skipped++; else done++; - updateJob(jobId, { progress: Math.round(((done + skipped) / total) * 90) }); - } catch (e) { - log(jobId, `⚠️ Fehler bei "${track.trackName || track.title}": ${e.message}`); - } - } - return { done, skipped }; -} - -// Search iTunes for album tracks by artist + album name (fallback when no collectionId) -async function searchItunesAlbumTracks(artist, album) { - return new Promise(resolve => { - const q = encodeURIComponent(`${artist} ${album}`); - https.get(`https://itunes.apple.com/search?term=${q}&media=music&entity=album&limit=5&country=us&explicit=Yes`, res => { - let d = ''; - res.on('data', c => d += c); - res.on('end', () => { - try { - const results = JSON.parse(d).results || []; - const match = results.find(r => - r.collectionName?.toLowerCase().includes(album.toLowerCase()) || - r.artistName?.toLowerCase().includes(artist.toLowerCase()) - ); - if (match) fetchItunesTracks(match.collectionId).then(resolve).catch(() => resolve([])); - else resolve([]); - } catch { resolve([]); } - }); - }).on('error', () => resolve([])); - }); -} - -// Fetch individual tracks for an iTunes collectionId -async function fetchItunesTracks(collectionId) { - return new Promise((resolve, reject) => { - const url = `https://itunes.apple.com/lookup?id=${collectionId}&entity=song&explicit=Yes`; - https.get(url, res => { - let data = ''; - res.on('data', d => data += d); - res.on('end', () => { - try { - const parsed = JSON.parse(data); - resolve((parsed.results || []).filter(r => r.wrapperType === 'track')); - } catch { reject(new Error('iTunes API parse error')); } - }); - }).on('error', reject); - }); -} - // ─── QUEUE ──────────────────────────────────────────────────────────────────── let activeDownloads = 0; const downloadQueue = []; @@ -564,11 +61,7 @@ function enqueue(task) { } // ─── SPOTIFY ────────────────────────────────────────────────────────────────── -// Scrapes Spotify's public embed pages — no token, no account, no Premium needed. -// Embed pages serve __NEXT_DATA__ JSON to anyone (used for website embeds worldwide). - function parseSpotifyUrl(url) { - // Handles locale prefixes: /intl-de/track/... and plain /track/... const m = url.match(/open\.spotify\.com\/(?:[a-z]{2,8}-[a-z]{2,8}\/)?(?:intl-[a-z-]+\/)?(track|album|playlist)\/([A-Za-z0-9]+)/); if (!m) throw new Error('Ungültige Spotify-URL — erwartet: open.spotify.com/track|album|playlist/...'); return { type: m[1], id: m[2] }; @@ -593,13 +86,11 @@ async function fetchSpotifyEmbed(type, id) { || nd?.props?.pageProps?.entity || nd?.props?.pageProps?.state?.data; if (!entity) throw new Error('Keine Metadaten in Spotify-Embed gefunden'); - // Unwrap GraphQL-style data wrapper: {data: {__typename, name, ...}} if (entity.data && !entity.name && !entity.trackList) entity = entity.data; return entity; } function _spArtwork(entity) { - // Handle albumOfTrack (newer GraphQL embed format) + legacy album/coverArt paths const sources = entity?.albumOfTrack?.coverArt?.sources || entity?.album?.coverArt?.sources || entity?.coverArt?.sources @@ -613,7 +104,6 @@ function _spArtwork(entity) { function _spAllArtists(artistsRaw) { if (!artistsRaw) return ''; - // Handle both array format and GraphQL {items:[...]} format const arr = Array.isArray(artistsRaw) ? artistsRaw : (artistsRaw.items || []); return arr.map(a => a?.profile?.name || a?.name || '').filter(Boolean).join(', '); } @@ -623,29 +113,20 @@ function _spArtistName(artistsRaw) { } function _spTrackItem(t, defaultAlbum, defaultArt, defaultAlbumArtist) { - // Unwrap optional {track: ...} wrapper (old playlist format) let raw = t?.track || t; - // Handle GraphQL-style {data: {__typename, ...}} const d = raw?.data || raw; - - // Name: new embed uses "title", old/track embed uses "name" const trackName = d?.name || d?.title || ''; - - // Artists: array format [{name}] or [{profile:{name}}], GraphQL {items:[...]}, - // or flat string in "subtitle" (new playlist embed: "Myrne, Aviella") let artist = ''; if (d?.artists) { artist = _spAllArtists(d.artists); } else if (typeof d?.subtitle === 'string' && d.subtitle) { - artist = d.subtitle; // already comma-separated, matches iTunes/Spotify standard + artist = d.subtitle; } artist = artist || defaultAlbumArtist || ''; - - const art = _spArtwork(d) || defaultArt || ''; + const art = _spArtwork(d) || defaultArt || ''; const albumObj = d?.albumOfTrack || d?.album || {}; - const album = albumObj?.name || defaultAlbum || ''; - const date = albumObj?.date?.isoString || albumObj?.release_date || ''; - + const album = albumObj?.name || defaultAlbum || ''; + const date = albumObj?.date?.isoString || albumObj?.release_date || ''; return { trackName, artistName: artist, @@ -679,7 +160,15 @@ app.post('/api/spotify/track-artwork', async (req, res) => { }); app.get('/api/health', (req, res) => { - res.json({ status: 'ok', ytdlpVersion, musicPath: CONFIG.musicPath, audioFormat: CONFIG.audioFmt, activeJobs: activeDownloads, queueLength: downloadQueue.length }); + res.json({ + status: 'ok', ytdlpVersion, + musicPath: CONFIG.musicPath, + audioFormat: CONFIG.audioFmt, + downloadBackend: CONFIG.downloadBackend, + squidQuality: CONFIG.squidQuality, + activeJobs: activeDownloads, + queueLength: downloadQueue.length, + }); }); app.post('/api/spotify/resolve', async (req, res) => { @@ -688,7 +177,6 @@ app.post('/api/spotify/resolve', async (req, res) => { try { const { type, id } = parseSpotifyUrl(url); const entity = await fetchSpotifyEmbed(type, id); - // "subtitle" is used by the flat embed format as artist/owner string const artist = _spArtistName(entity.artists) || (typeof entity.subtitle === 'string' ? entity.subtitle : ''); const artUrl = _spArtwork(entity); @@ -726,8 +214,6 @@ app.post('/api/spotify/resolve', async (req, res) => { artist: owner, artworkUrl: artUrl, description: entity.description || '', - // no defaultArt for playlist tracks — iTunes lazy-loader will fill correct per-track art; - // passing playlist cover here would cause it to be embedded in downloads for unresolved tracks tracks: trackList.map(t => _spTrackItem(t, null, null, null)), }); } @@ -756,20 +242,20 @@ app.post('/api/download', async (req, res) => { updateJob(jobId, { status: 'running', progress: 5, message: 'Gestartet…' }); try { if (type === 'track') { - // Prefer Spotify metadata from tracks[0] when available (more accurate than iTunes lookup) - const sp = req.body.tracks?.[0] || {}; + const sp = req.body.tracks?.[0] || {}; const spAlbum = sp.collectionName || album || ''; - const spArt = sp.artworkUrl100 || artworkUrl || ''; + const spArt = sp.artworkUrl100 || artworkUrl || ''; const spYear = sp.releaseDate ? new Date(sp.releaseDate).getFullYear() : undefined; - let meta = { album: spAlbum, year: spYear, + let meta = { + album: spAlbum, year: spYear, trackNum: sp.trackNumber || bodyTrackNum || undefined, totalTracks: sp.trackCount || bodyTotalTracks || undefined, discNum: sp.discNumber || bodyDiscNum || undefined, discTotal: sp.discCount || bodyDiscTotal || undefined, artworkUrl: spArt, explicit: sp.trackExplicitness === 'explicit' || !!explicit, - durationMs: sp.durationMs || undefined }; + durationMs: sp.durationMs || undefined, + }; if (!meta.album) { - // Use first artist only for iTunes search (multi-artist strings like "Kløn, Renzyx" confuse iTunes) const searchArtist = artist.split(',')[0].trim(); log(jobId, '📋 Suche Album-Metadaten von iTunes…'); const extra = await lookupItunesTrackMeta(searchArtist, title); @@ -799,7 +285,7 @@ app.post('/api/download', async (req, res) => { throw new Error(`Keine einzelnen Tracks für "${album || title}" gefunden. Bitte über iTunes Global-Suche herunterladen.`); } const { done: dlDone, skipped: dlSkipped } = await downloadAlbumTracks({ jobId, artist, albumTitle: album || title, tracks, artworkUrl }); - const allSkipped = dlSkipped === tracks.length; + const allSkipped = dlSkipped === tracks.length; const someSkipped = dlSkipped > 0 && !allSkipped; const msg = allSkipped ? `Album bereits vorhanden – alle ${dlSkipped} Tracks übersprungen` @@ -812,7 +298,8 @@ app.post('/api/download', async (req, res) => { log(jobId, `🎤 Künstler-Download: ${artist}`); const artistData = await new Promise((resolve, reject) => { https.get(`https://itunes.apple.com/search?term=${encodeURIComponent(artist)}&media=music&entity=album&limit=10&country=de`, res => { - let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d).results || []); } catch { reject(new Error('parse')); } }); + let d = ''; res.on('data', c => d += c); + res.on('end', () => { try { resolve(JSON.parse(d).results || []); } catch { reject(new Error('parse')); } }); }).on('error', reject); }); const artistAlbums = artistData.filter(a => a.artistName?.toLowerCase() === artist.toLowerCase()).slice(0, 10); @@ -837,9 +324,8 @@ app.post('/api/download', async (req, res) => { for (let i = 0; i < plTracks.length; i += CONC) { const batch = plTracks.slice(i, i + CONC); const batchResults = await Promise.allSettled(batch.map(async track => { - const trackTitle = track.trackName || ''; - // subtitle gives "Artist1, Artist2" — use first for iTunes search, keep full for tags - const trackArtist = track.artistName || artist || ''; + const trackTitle = track.trackName || ''; + const trackArtist = track.artistName || artist || ''; const searchArtist = trackArtist.split(',')[0].trim(); log(jobId, `📋 iTunes: "${trackArtist} – ${trackTitle}"`); const itunes = await lookupItunesTrackMeta(searchArtist, trackTitle); @@ -898,8 +384,12 @@ app.get('/api/jobs', (req, res) => { // ─── START ──────────────────────────────────────────────────────────────────── app.listen(CONFIG.port, () => { console.log(`\n🎵 NovaSonic Backend running on port ${CONFIG.port}`); - console.log(` Music folder : ${CONFIG.musicPath}`); - console.log(` Audio format : ${CONFIG.audioFmt} (quality ${CONFIG.audioQuality})`); - console.log(` yt-dlp : ${ytdlpVersion}`); + console.log(` Music folder : ${CONFIG.musicPath}`); + console.log(` Audio format : ${CONFIG.audioFmt} (quality ${CONFIG.audioQuality})`); + console.log(` Download backend : ${CONFIG.downloadBackend}`); + if (CONFIG.downloadBackend !== 'ytdlp') { + console.log(` Squid quality : ${CONFIG.squidQuality}`); + } + console.log(` yt-dlp : ${ytdlpVersion}`); console.log(''); }); diff --git a/docker-compose.yml b/docker-compose.yml index d0ab856..e37e113 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: restart: unless-stopped working_dir: /app volumes: - - ./backend/novasonic-server.js:/app/novasonic-server.js:ro + - ./backend:/app:ro # mount entire backend dir (server + lib/) - ${MUSIC_PATH}:${MUSIC_PATH} # exact path from .env, no remapping ports: - "3700:3700" # route this port through your reverse proxy @@ -28,9 +28,11 @@ services: - AUDIO_FMT=${AUDIO_FMT:-mp3} - AUDIO_QUALITY=${AUDIO_QUALITY:-0} - MAX_PARALLEL=${MAX_PARALLEL:-2} + - DOWNLOAD_BACKEND=${DOWNLOAD_BACKEND:-auto} + - SQUID_QUALITY=${SQUID_QUALITY:-mp3} command: > sh -c " apk add --no-cache yt-dlp ffmpeg python3 && - npm install express cors && - node novasonic-server.js + mkdir -p /deps && cd /deps && npm install express && + cd /app && NODE_PATH=/deps/node_modules node novasonic-server.js "