Files
NovaSonic/backend/lib/ytdlp.js
T
HoneyandClaude Sonnet 4.6 3c54850cca fix(backend): add ytmsearch5 step + expand YTM pool to 20 for better coverage
ytmsearch5 (yt-dlp native YTM search) added as step 2 — returns different
result ordering than the manual YTM URL, recovering songs that the URL
approach ranks too low. YTM pool expanded from 10 → 20 results to reduce
fallthrough to YouTube.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-19 10:25:37 +02:00

436 lines
22 KiB
JavaScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');
const CONFIG = require('./config');
const { log } = require('./jobs');
const {
sanitize, navidromeArtist,
downloadArtwork, findExistingTrack,
} = require('./metadata');
const { romajiForStorage, isTokenizerReady } = require('./romaji');
// Matches hiragana, katakana, CJK — used to detect Japanese content
const JP_RE = /[ぁ-んァ-ン一-龯]/;
function buildTitleKeywords(title) {
// For Japanese titles: \w strips all kana/kanji, leaving nothing useful.
// Instead use the first whitespace-delimited segment as a literal keyword so
// the YTM --match-filter requires the video title to contain it.
if (JP_RE.test(title)) {
const firstWord = (title.split(/[\s ]/)[0] || '').slice(0, 12);
const escaped = firstWord.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
return escaped ? [escaped] : [];
}
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with','single','ep','album','remaster','remastered','deluxe','edition','version','live','bonus']);
const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
const lower = (title || '').toLowerCase();
const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim();
const words = strippedTitle
.replace(/[^\w\s]/g, ' ')
.split(/\s+/)
.filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase()));
const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2);
const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w;
const result = top2.map(toPattern);
const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null;
if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier);
return result;
}
function buildVariantFilter(title) {
const t = (title || '').toLowerCase();
// !~= (?i) = case-insensitive regex "does not match". Use \b word boundaries so that
// partial matches like "nightcoreals" containing "nightcore" don't falsely reject results.
const RULES = [
{ key: 'sped', expr: 'title !~= (?i)\\bsped\\b' },
{ key: 'speedup', expr: 'title !~= (?i)\\bspeed.{0,2}up\\b' },
{ key: 'slowed', expr: 'title !~= (?i)\\bslowed\\b' },
{ key: 'reverb', expr: 'title !~= (?i)\\breverb\\b' },
{ key: 'nightcore', expr: 'title !~= (?i)\\bnightcore\\b' },
{ key: 'karaoke', expr: 'title !~= (?i)\\bkaraoke\\b' },
{ key: 'parody', expr: 'title !~= (?i)\\bparody\\b' },
{ key: 'remix', expr: 'title !~= (?i)\\bremix\\b' },
{ key: 'cover', expr: 'title !~= (?i)\\bcover\\b' },
{ key: 'acoustic', expr: 'title !~= (?i)\\bacoustic\\b' },
{ key: 'instrumental', expr: 'title !~= (?i)\\binstrumental\\b' },
];
const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr);
return parts.length ? parts.join(' & ') : '';
}
async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs, sourceUrl }) {
const mainArtist = albumArtist || artist;
// If the title or artist contains Japanese characters, wait for kuromoji to finish loading.
// Without it romajiForStorage returns the original text unchanged, producing Japanese
// folder/filenames and preventing sort tags from being written.
if (JP_RE.test(title) || JP_RE.test(rawArtist || artist || '')) {
if (!isTokenizerReady()) {
log(jobId, '⏳ Warte auf kuromoji-Tokenizer…');
await new Promise(resolve => {
const t0 = Date.now();
const iv = setInterval(() => {
if (isTokenizerReady() || Date.now() - t0 > 30000) { clearInterval(iv); resolve(); }
}, 300);
});
}
}
// Folder / filename: romaji of the resolved artist (iTunes name if available — already in English,
// otherwise kuromoji). rawArtist (original Japanese) is only used for tags + YTM search.
const mainArtistRomaji = romajiForStorage(mainArtist) || mainArtist;
const artistRomaji = romajiForStorage(artist) || artist;
const titleRomaji = romajiForStorage(title) || title;
const albumRomaji = album ? (romajiForStorage(album) || album) : null;
const safeArtist = sanitize(mainArtistRomaji.split(/[,;&]/)[0].trim());
const safeAlbum = sanitize(albumRomaji || 'Singles');
const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum);
fs.mkdirSync(dir, { recursive: true });
const safeTitle = sanitize(titleRomaji);
const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : '';
// Filename format: "{01 - }Artist1, Artist2 - Title.ext"
// Use track artist (not albumArtist) so all credited artists appear in the filename
const safeArtistForFile = sanitize(artistRomaji.replace(/;/g, ','));
const fileBase = `${trackPrefix}${safeArtistForFile} - ${safeTitle}`;
const existing = findExistingTrack(dir, trackPrefix, safeTitle, safeArtistForFile);
if (existing) {
log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`);
return { success: true, dir, skipped: true };
}
const outputTemplate = path.join(dir, `${fileBase}.%(ext)s`);
// For YTM search: use the original (raw) artist when available — the YTM channel is registered
// under the Japanese/original name, not the iTunes-resolved English name.
// rawArtist is undefined for non-Japanese tracks, so fall back to mainArtist.
const ytmArtist = (rawArtist || mainArtist).replace(/\p{S}/gu, '').trim();
const searchArtist = ytmArtist;
const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
// Strip release-type suffixes Spotify appends to track names (e.g. "recognize - Single" → "recognize").
// Only strip trailing "- word(s)" when those words are release-type qualifiers or bare years, not
// real sub-titles (which tend to be longer phrases or start with "feat"/"ft").
const RELEASE_SUFFIX_RE = /\s*[-–]\s*(?:(?:official\s+)?(?:music\s+)?(?:single|ep|lp|album|ost|soundtrack|remaster(?:ed)?|deluxe(?:\s+edition)?|bonus\s+tracks?|b-?sides?|version|edit)\b.*|(?:20|19)\d{2})\s*$/i;
const searchTitle = title.replace(RELEASE_SUFFIX_RE, '').trim() || title;
const variantFilter = buildVariantFilter(searchTitle);
const titleKws = buildTitleKeywords(searchTitle);
const variantPart = variantFilter ? ` & ${variantFilter}` : '';
const titleKwPart = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : '';
// Soft channel check: first word of original artist name used in early cascade steps to avoid
// downloading a same-title song by a different artist (e.g. wrong "Moonlight" for N!GHT).
// Skip when artistFirstWord is non-ASCII (Japanese kanji/kana): YouTube channel names are
// almost never in kanji, so the filter would reject every valid result.
const artistFirstWord = (rawArtist || mainArtist).split(/[\s,/]+/)[0];
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_RE.test(artistFirstWord))
? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';
// When the title is Japanese (keywords stripped to nothing) OR when we have an album that
// provides better search context, include the album's first two words in the search query.
// This helps YTM rank the official OST/album track above fan covers and romaji-subtitle uploads.
// Only use album words that are ASCII (romaji/English) — Japanese album words don't help YTM ranking.
const hasJpTitle = JP_RE.test(title);
const GENERIC_ALBUM_WORDS = new Set(['single', 'ep', 'album', 'soundtrack', 'ost', 'edition', 'remaster', 'remastered', 'deluxe', 'bonus', 'version', 'live']);
const titleWordSet = new Set(title.toLowerCase().split(/\s+/));
const albumWords = (album && album !== 'Singles' && album !== 'Unknown')
? album.split(/\s+/)
.filter(w => /^[A-Za-z0-9]/.test(w) && !titleWordSet.has(w.toLowerCase()) && !GENERIC_ALBUM_WORDS.has(w.toLowerCase()))
.slice(0, 2).join(' ')
: '';
// Only append album words for Japanese titles where title keywords are stripped to nothing.
// For Western tracks, artist + title + duration matching is sufficient (matches spotdl behaviour).
const albumQuerySuffix = hasJpTitle && albumWords ? ' ' + albumWords : '';
const query = `${ytmArtist} - ${searchTitle}${albumQuerySuffix}`;
const baseArgs = [
'--extract-audio',
'--audio-format', CONFIG.audioFmt,
'--audio-quality', CONFIG.audioQuality,
'-o', outputTemplate,
'--no-playlist',
'--no-warnings',
'--newline',
];
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
let stderr = '', lastPct = '';
proc.stdout.on('data', d => {
d.toString().split('\n').filter(l => l.trim()).forEach(line => {
if (line.includes('%') && line === lastPct) return;
if (line.includes('%')) lastPct = line;
log(jobId, line.trim());
});
});
proc.stderr.on('data', d => {
stderr += d.toString();
d.toString().split('\n')
.filter(l => l.trim() && !l.startsWith('[debug]'))
.forEach(l => log(jobId, l.trim()));
});
proc.on('error', resolve);
proc.on('close', code => {
// Treat all non-zero exits as "no result" so the cascade continues.
// Fatal errors (network, bad args) are logged via stderr above; hasAudio() decides outcome.
if (code !== 0 && code !== 101) log(jobId, `⚠️ yt-dlp exit ${code}: ${stderr.slice(-200).trim()}`);
resolve();
});
});
const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D';
// Use original (raw) artist for YTM URL — better ranking since channel is in original language
const ytmQuery = `${rawArtist || mainArtist} - ${searchTitle}${albumQuerySuffix}`;
const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(ytmQuery)}&params=${YTM_SONGS_PARAM}`;
const expectedAudio = path.join(dir, `${fileBase}.${CONFIG.audioFmt}`);
const hasAudio = () => fs.existsSync(expectedAudio);
// Direct URL download (e.g. SoundCloud) — skip the ytsearch cascade entirely
if (sourceUrl) {
log(jobId, `🔗 Direkter Download: ${sourceUrl}`);
await spawnYtdlp([sourceUrl, '--no-playlist', ...baseArgs]);
const audioFile = hasAudio() ? expectedAudio : null;
if (!audioFile) { log(jobId, '⚠️ Download fehlgeschlagen'); return { success: true, dir }; }
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
}
// Fetch YTM search result URLs as a flat list (no channel expansion) and return only
// actual video URLs (watch?v=…). This prevents yt-dlp from following an artist channel
// result and treating its uploads as the search result set.
//
// yt-dlp expands "url_transparent" artist channel entries even under --flat-playlist,
// printing the channel's video URLs instead of the channel URL. We detect this by
// watching for [youtube:tab] in output (which only fires during channel/tab extraction)
// and return [] so the caller falls through to the ytsearch fallback.
const getYtmVideoUrls = (maxItems) => new Promise(resolve => {
const urls = [];
let channelExpanded = false;
const checkExpansion = (text) => {
if (text.includes('[youtube:tab]') || text.includes('/browse/UC')) channelExpanded = true;
};
const proc = spawn(CONFIG.ytdlpBin, [
ytmUrl,
'--flat-playlist',
'--playlist-items', `1-${maxItems}`,
'--print', 'webpage_url',
'--no-warnings',
], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
proc.stdout.on('data', d => {
const text = d.toString();
checkExpansion(text);
if (!channelExpanded) {
text.split('\n').forEach(line => {
const u = line.trim();
if (u.includes('watch?v=')) urls.push(u);
});
}
});
proc.stderr.on('data', d => checkExpansion(d.toString()));
proc.on('error', () => resolve([]));
proc.on('close', () => resolve(channelExpanded ? [] : urls));
});
const mf = (dur, variant, strict = false) => {
const durPart = dur ? `duration > ${dur.min} & duration < ${dur.max} & ` : '';
return ['--match-filter',
`${durPart}!is_live${titleKwPart}${strict ? artistChannelPart : ''}${variant ? variantPart : ''}`];
};
const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1'];
const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
const PI_UNFILTERED = ['--playlist-items', '1'];
// Fetch YTM URLs once upfront so every cascade step filters the SAME result set.
// Re-fetching produced different orderings, causing the correct video (found in step 1
// but failing the channel filter) to disappear by step 3 and a wrong one to slip through.
const ytmUrls = await getYtmVideoUrls(20);
const tryYtm = async (n, mfArgs) => {
const urls = n ? ytmUrls.slice(0, n) : ytmUrls;
if (!urls.length) return;
await spawnYtdlp([...urls, '--max-downloads', '1', ...mfArgs, ...baseArgs]);
};
if (durationMs && durationMs > 30000) {
const durationSec = Math.round(durationMs / 1000);
const margin = Math.max(10, Math.round(durationSec * 0.10));
const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };
log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
// 1. YTM top-5 via manual URL — strict: duration + title + variant + channel check
await tryYtm(5, mf(dur, true, true));
if (!hasAudio()) {
// 2. ytmsearch5 (yt-dlp native YTM) — different result ordering than manual URL
await spawnYtdlp([`ytmsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true, true), ...baseArgs]);
}
if (!hasAudio()) {
// 3. YT top-5 — strict: same filters including channel check
log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…');
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true, true), ...baseArgs]);
}
if (!hasAudio()) {
// 4. YTM all-20 — no channel check (label channels, etc.)
log(jobId, '⚠️ Erweitere Suche auf top 20 mit Dauerbeschränkung…');
await tryYtm(0, mf(dur, true));
}
if (!hasAudio()) {
// 5. YT top-10 — no channel check
await spawnYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
}
if (!hasAudio()) {
// 6. YTM all — no duration check
log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…');
await tryYtm(0, mf(null, true));
}
if (!hasAudio()) {
// 7. YT top-5 — no duration check
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
}
if (!hasAudio()) {
// 8. YTM all — title only (no variant filter)
log(jobId, '⚠️ Letzter gefilterter Versuch…');
await tryYtm(0, mf(null, false));
}
if (!hasAudio()) {
// 9. YT top-5 — title only
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
}
if (!hasAudio()) {
// 10. YTM — no filter at all, take first video result
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
await tryYtm(3, []);
}
if (!hasAudio()) {
// 11. YT — no filter
await spawnYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
}
} else {
log(jobId, `🔍 Suche YT Music: "${query}"`);
// 1. YTM top-5 — strict: title + variant + channel check
await tryYtm(5, mf(null, true, true));
if (!hasAudio()) {
// 2. YT top-5 — strict
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true, true), ...baseArgs]);
}
if (!hasAudio()) {
// 3. YTM all-10 — no channel check (catches label/distributor channels)
await tryYtm(0, mf(null, true));
}
if (!hasAudio()) {
// 4. YT top-5 — no channel check
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
}
if (!hasAudio()) {
// 5. YTM all — title only
log(jobId, '⚠️ Letzter gefilterter Versuch…');
await tryYtm(0, mf(null, false));
}
if (!hasAudio()) {
// 6. YT top-5 — title only
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
}
if (!hasAudio()) {
// 7. YTM — first video result, no filter
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
await tryYtm(3, []);
}
if (!hasAudio()) {
// 8. YT — no filter
await spawnYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
}
}
const audioFile = hasAudio() ? expectedAudio : null;
if (!audioFile) {
log(jobId, '⚠️ Keine Audio-Datei gefunden');
return { success: true, dir };
}
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
}
// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
async function embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title }) {
const coverPath = await downloadArtwork(artworkUrl, dir);
const hasCover = !!(coverPath && fs.existsSync(coverPath));
log(jobId, `🏷 Metadaten + Cover → ${path.basename(audioFile)}`);
const fileExt = path.extname(audioFile);
const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt;
await new Promise(resolve => {
const ffArgs = ['-i', audioFile];
if (hasCover) ffArgs.push('-i', coverPath);
ffArgs.push('-map', '0:a');
if (hasCover) ffArgs.push('-map', '1:v');
// Main tags: store original text (Japanese or whatever was passed in).
// rawArtist = original Spotify/source artist (may be Japanese); artist = iTunes-resolved (may be English).
// Use rawArtist for the stored tag so Apple Music / other apps see the authentic original.
const tagArtist = navidromeArtist(rawArtist || artist);
const tagAlbumArtist = (rawArtist || mainArtist).replace(/\s+&\s+/g, ', ');
// Sort tags: romaji for display / sorting in apps that support them (TITLESORT, ARTISTSORT, etc.)
// artist is already the iTunes-resolved English name when available — use directly (more accurate
// than kuromoji). For non-iTunes tracks romajiForStorage acts as kuromoji fallback.
const sortTitle = romajiForStorage(title) || title;
const sortArtist = romajiForStorage(navidromeArtist(artist)) || navidromeArtist(artist);
const sortAlbumArtist = romajiForStorage(mainArtist.replace(/\s+&\s+/g, ', ')) || mainArtist.replace(/\s+&\s+/g, ', ');
const sortAlbum = album ? (romajiForStorage(album) || album) : null;
ffArgs.push(
'-c', 'copy',
'-map_metadata', '-1',
'-id3v2_version', '3',
'-metadata', `title=${title}`,
'-metadata', `artist=${tagArtist}`,
'-metadata', `album_artist=${tagAlbumArtist}`,
);
if (album) ffArgs.push('-metadata', `album=${album}`);
// Sort tags — only write when they differ from main (no-op for non-Japanese tracks)
if (sortTitle !== title) ffArgs.push('-metadata', `TITLESORT=${sortTitle}`);
if (sortArtist !== tagArtist) ffArgs.push('-metadata', `ARTISTSORT=${sortArtist}`);
if (sortAlbumArtist !== tagAlbumArtist) ffArgs.push('-metadata', `ALBUMARTISTSORT=${sortAlbumArtist}`);
if (sortAlbum && sortAlbum !== album) ffArgs.push('-metadata', `ALBUMSORT=${sortAlbum}`);
if (year) ffArgs.push('-metadata', `date=${String(year)}`);
if (trackNum) ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`);
if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`);
if (genre) ffArgs.push('-metadata', `genre=${genre}`);
if (discNum) ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`);
if (compilation) ffArgs.push('-metadata', `compilation=1`);
if (explicit) ffArgs.push('-metadata', 'ITUNESADVISORY=1');
if (hasCover) {
ffArgs.push(
'-metadata:s:v', 'title=Album cover',
'-metadata:s:v', 'comment=Cover (front)',
);
}
ffArgs.push(tmpFile, '-y');
const ff = spawn('ffmpeg', ffArgs);
let ffErr = '';
ff.stderr.on('data', d => ffErr += d.toString());
ff.on('error', () => resolve());
ff.on('close', code => {
if (code === 0 && fs.existsSync(tmpFile)) {
fs.renameSync(tmpFile, audioFile);
if (hasCover) try { fs.unlinkSync(coverPath); } catch {}
const base = path.basename(audioFile, path.extname(audioFile));
for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) {
try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {}
}
log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`);
} else {
try { fs.unlinkSync(tmpFile); } catch {}
log(jobId, `⚠️ ffmpeg code ${code}`);
if (ffErr) log(jobId, ffErr.slice(-200));
}
resolve();
});
});
return { success: true, dir };
}
module.exports = { downloadTrack, embedMetadata };