HoneyandClaude Sonnet 4.6
03fb45b986
- YTM URL now uses unstripped artist name (e.g. "N!GHT" instead of "NGHT") for better search ranking on YouTube Music - Steps 1-2 of cascade now include a soft channel ~= filter using the artist's first word, preventing wrong-artist songs with matching titles from being downloaded (e.g. a different "Moonlight" for N!GHT) - Steps 3+ drop the channel filter so label-hosted tracks still get found Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
348 lines
15 KiB
JavaScript
348 lines
15 KiB
JavaScript
const { spawn } = require('child_process');
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const CONFIG = require('./config');
|
|
const { log } = require('./jobs');
|
|
const {
|
|
sanitize, navidromeArtist,
|
|
downloadArtwork, fetchLyrics, findExistingTrack,
|
|
} = require('./metadata');
|
|
|
|
function buildTitleKeywords(title) {
|
|
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
|
|
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
|
|
const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
|
|
const lower = (title || '').toLowerCase();
|
|
const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim();
|
|
const words = strippedTitle
|
|
.replace(/[^\w\s]/g, ' ')
|
|
.split(/\s+/)
|
|
.filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase()));
|
|
const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2);
|
|
const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w;
|
|
const result = top2.map(toPattern);
|
|
const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null;
|
|
if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier);
|
|
return result;
|
|
}
|
|
|
|
function buildVariantFilter(title) {
|
|
const t = (title || '').toLowerCase();
|
|
// !~= (?i) = case-insensitive regex "does not match". Use \b word boundaries so that
|
|
// partial matches like "nightcoreals" containing "nightcore" don't falsely reject results.
|
|
const RULES = [
|
|
{ key: 'sped', expr: 'title !~= (?i)\\bsped\\b' },
|
|
{ key: 'speedup', expr: 'title !~= (?i)\\bspeed.{0,2}up\\b' },
|
|
{ key: 'slowed', expr: 'title !~= (?i)\\bslowed\\b' },
|
|
{ key: 'reverb', expr: 'title !~= (?i)\\breverb\\b' },
|
|
{ key: 'nightcore', expr: 'title !~= (?i)\\bnightcore\\b' },
|
|
{ key: 'karaoke', expr: 'title !~= (?i)\\bkaraoke\\b' },
|
|
{ key: 'parody', expr: 'title !~= (?i)\\bparody\\b' },
|
|
{ key: 'remix', expr: 'title !~= (?i)\\bremix\\b' },
|
|
{ key: 'cover', expr: 'title !~= (?i)\\bcover\\b' },
|
|
{ key: 'acoustic', expr: 'title !~= (?i)\\bacoustic\\b' },
|
|
{ key: 'instrumental', expr: 'title !~= (?i)\\binstrumental\\b' },
|
|
];
|
|
const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr);
|
|
return parts.length ? parts.join(' & ') : '';
|
|
}
|
|
|
|
async function downloadTrack({ jobId, artist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs }) {
|
|
const mainArtist = albumArtist || artist;
|
|
const safeArtist = sanitize(mainArtist.split(',')[0].trim());
|
|
const safeAlbum = sanitize(album || 'Singles');
|
|
const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum);
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
|
|
const safeTitle = sanitize(title);
|
|
const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : '';
|
|
|
|
const existing = findExistingTrack(dir, trackPrefix, safeTitle);
|
|
if (existing) {
|
|
log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`);
|
|
return { success: true, dir, skipped: true };
|
|
}
|
|
|
|
const outputTemplate = path.join(dir, `${trackPrefix}${safeTitle}.%(ext)s`);
|
|
const searchArtist = mainArtist.replace(/[^\p{L}\p{N}\s,&\-]/gu, '').trim();
|
|
const query = `${searchArtist} - ${title}`;
|
|
|
|
const baseArgs = [
|
|
'--extract-audio',
|
|
'--audio-format', CONFIG.audioFmt,
|
|
'--audio-quality', CONFIG.audioQuality,
|
|
'-o', outputTemplate,
|
|
'--no-playlist',
|
|
'--no-warnings',
|
|
'--newline',
|
|
];
|
|
|
|
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
|
|
const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
|
let stderr = '', lastPct = '';
|
|
proc.stdout.on('data', d => {
|
|
d.toString().split('\n').filter(l => l.trim()).forEach(line => {
|
|
if (line.includes('%') && line === lastPct) return;
|
|
if (line.includes('%')) lastPct = line;
|
|
log(jobId, line.trim());
|
|
});
|
|
});
|
|
proc.stderr.on('data', d => {
|
|
stderr += d.toString();
|
|
d.toString().split('\n')
|
|
.filter(l => l.trim() && !l.startsWith('[debug]'))
|
|
.forEach(l => log(jobId, l.trim()));
|
|
});
|
|
proc.on('error', reject);
|
|
proc.on('close', code =>
|
|
(code === 0 || code === 101) ? resolve() : reject(new Error(`yt-dlp code ${code}: ${stderr.slice(-300)}`))
|
|
);
|
|
});
|
|
|
|
const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D';
|
|
// Use original artist name (with special chars) for YTM URL — better search ranking than stripped version
|
|
const ytmQuery = `${mainArtist} - ${title}`;
|
|
const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(ytmQuery)}¶ms=${YTM_SONGS_PARAM}`;
|
|
const expectedAudio = path.join(dir, `${trackPrefix}${safeTitle}.${CONFIG.audioFmt}`);
|
|
const hasAudio = () => fs.existsSync(expectedAudio);
|
|
const tryYtdlp = async (args) => { await spawnYtdlp(args); };
|
|
|
|
// Fetch YTM search result URLs as a flat list (no channel expansion) and return only
|
|
// actual video URLs (watch?v=…). This prevents yt-dlp from following an artist channel
|
|
// result and treating its uploads as the search result set.
|
|
//
|
|
// yt-dlp expands "url_transparent" artist channel entries even under --flat-playlist,
|
|
// printing the channel's video URLs instead of the channel URL. We detect this by
|
|
// watching for [youtube:tab] in output (which only fires during channel/tab extraction)
|
|
// and return [] so the caller falls through to the ytsearch fallback.
|
|
const getYtmVideoUrls = (maxItems) => new Promise(resolve => {
|
|
const urls = [];
|
|
let channelExpanded = false;
|
|
const checkExpansion = (text) => {
|
|
if (text.includes('[youtube:tab]') || text.includes('/browse/UC')) channelExpanded = true;
|
|
};
|
|
const proc = spawn(CONFIG.ytdlpBin, [
|
|
ytmUrl,
|
|
'--flat-playlist',
|
|
'--playlist-items', `1-${maxItems}`,
|
|
'--print', 'webpage_url',
|
|
'--no-warnings',
|
|
], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
|
proc.stdout.on('data', d => {
|
|
const text = d.toString();
|
|
checkExpansion(text);
|
|
if (!channelExpanded) {
|
|
text.split('\n').forEach(line => {
|
|
const u = line.trim();
|
|
if (u.includes('watch?v=')) urls.push(u);
|
|
});
|
|
}
|
|
});
|
|
proc.stderr.on('data', d => checkExpansion(d.toString()));
|
|
proc.on('error', () => resolve([]));
|
|
proc.on('close', () => resolve(channelExpanded ? [] : urls));
|
|
});
|
|
|
|
// Search YTM for video URLs (pre-filtered), then pass them all at once to yt-dlp.
|
|
// --max-downloads 1 makes yt-dlp stop after the first video that passes mfArgs.
|
|
// Filtered/skipped videos do NOT count toward --max-downloads, so the cascade works.
|
|
const tryYtmSearch = async (maxItems, mfArgs) => {
|
|
const urls = await getYtmVideoUrls(maxItems);
|
|
if (!urls.length) return;
|
|
await tryYtdlp([...urls, '--max-downloads', '1', ...mfArgs, ...baseArgs]);
|
|
};
|
|
|
|
const variantFilter = buildVariantFilter(title);
|
|
const titleKws = buildTitleKeywords(title);
|
|
const variantPart = variantFilter ? ` & ${variantFilter}` : '';
|
|
const titleKwPart = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : '';
|
|
|
|
// Soft channel check: first word of original artist name used in early cascade steps to avoid
|
|
// downloading a same-title song by a different artist (e.g. wrong "Moonlight" for N!GHT).
|
|
// Dropped in later fallback steps so genuine mismatches (label channels etc.) still get found.
|
|
const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
const artistFirstWord = mainArtist.split(/[\s,/]+/)[0];
|
|
const artistChannelPart = artistFirstWord.length >= 3
|
|
? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';
|
|
|
|
const mf = (dur, variant, strict = false) => {
|
|
const durPart = dur ? `duration > ${dur.min} & duration < ${dur.max} & ` : '';
|
|
return ['--match-filter',
|
|
`${durPart}!is_live${titleKwPart}${strict ? artistChannelPart : ''}${variant ? variantPart : ''}`];
|
|
};
|
|
|
|
const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1'];
|
|
const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
|
|
const PI_UNFILTERED = ['--playlist-items', '1'];
|
|
|
|
if (durationMs) {
|
|
const durationSec = Math.round(durationMs / 1000);
|
|
const margin = Math.max(15, Math.round(durationSec * 0.15));
|
|
const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };
|
|
|
|
log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
|
|
// 1. YTM top-5 — strict: duration + title + variant + channel check
|
|
await tryYtmSearch(5, mf(dur, true, true));
|
|
if (!hasAudio()) {
|
|
// 2. YT top-5 — strict: same filters including channel check
|
|
log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…');
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true, true), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 3. YTM top-10 — no channel check (label channels, etc.)
|
|
log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…');
|
|
await tryYtmSearch(10, mf(dur, true));
|
|
}
|
|
if (!hasAudio()) {
|
|
// 4. YT top-10 — no channel check
|
|
await tryYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 5. YTM top-5 — no duration check
|
|
log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…');
|
|
await tryYtmSearch(5, mf(null, true));
|
|
}
|
|
if (!hasAudio()) {
|
|
// 6. YT top-5 — no duration check
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 7. YTM top-5 — title only (no variant filter)
|
|
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
|
await tryYtmSearch(5, mf(null, false));
|
|
}
|
|
if (!hasAudio()) {
|
|
// 8. YT top-5 — title only
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 9. YTM — no filter at all, take first video result
|
|
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
|
await tryYtmSearch(3, []);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 10. YT — no filter
|
|
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
|
}
|
|
} else {
|
|
log(jobId, `🔍 Suche YT Music: "${query}"`);
|
|
// 1. YTM top-5 — strict: title + variant + channel check
|
|
await tryYtmSearch(5, mf(null, true, true));
|
|
if (!hasAudio()) {
|
|
// 2. YT top-5 — strict
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true, true), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 3. YTM top-5 — no channel check
|
|
await tryYtmSearch(5, mf(null, true));
|
|
}
|
|
if (!hasAudio()) {
|
|
// 4. YT top-5 — no channel check
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 5. YTM top-5 — title only
|
|
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
|
await tryYtmSearch(5, mf(null, false));
|
|
}
|
|
if (!hasAudio()) {
|
|
// 6. YT top-5 — title only
|
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 7. YTM — first video result, no filter
|
|
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
|
await tryYtmSearch(3, []);
|
|
}
|
|
if (!hasAudio()) {
|
|
// 8. YT — no filter
|
|
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
|
}
|
|
}
|
|
|
|
const audioFile = hasAudio() ? expectedAudio : null;
|
|
if (!audioFile) {
|
|
log(jobId, '⚠️ Keine Audio-Datei gefunden');
|
|
return { success: true, dir };
|
|
}
|
|
|
|
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
|
|
}
|
|
|
|
// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
|
|
async function embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title }) {
|
|
const [coverPath, lyricsData] = await Promise.all([
|
|
downloadArtwork(artworkUrl, dir),
|
|
fetchLyrics(artist, title, album),
|
|
]);
|
|
const hasCover = !!(coverPath && fs.existsSync(coverPath));
|
|
|
|
if (lyricsData?.synced) {
|
|
const lrcPath = path.join(dir, `${trackPrefix}${safeTitle}.lrc`);
|
|
fs.writeFileSync(lrcPath, lyricsData.synced, 'utf8');
|
|
log(jobId, '🎵 Synced lyrics gefunden');
|
|
} else if (lyricsData?.plain) {
|
|
log(jobId, '📝 Lyrics gefunden');
|
|
}
|
|
|
|
log(jobId, `🏷 Metadaten + Cover → ${path.basename(audioFile)}`);
|
|
|
|
const fileExt = path.extname(audioFile);
|
|
const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt;
|
|
await new Promise(resolve => {
|
|
const ffArgs = ['-i', audioFile];
|
|
if (hasCover) ffArgs.push('-i', coverPath);
|
|
ffArgs.push('-map', '0:a');
|
|
if (hasCover) ffArgs.push('-map', '1:v');
|
|
ffArgs.push(
|
|
'-c', 'copy',
|
|
'-map_metadata', '-1',
|
|
'-id3v2_version', '3',
|
|
'-metadata', `title=${title}`,
|
|
'-metadata', `artist=${navidromeArtist(artist)}`,
|
|
'-metadata', `album_artist=${mainArtist}`,
|
|
);
|
|
if (album) ffArgs.push('-metadata', `album=${album}`);
|
|
if (year) ffArgs.push('-metadata', `date=${String(year)}`);
|
|
if (trackNum) ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`);
|
|
if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`);
|
|
if (genre) ffArgs.push('-metadata', `genre=${genre}`);
|
|
if (discNum) ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`);
|
|
if (compilation) ffArgs.push('-metadata', `compilation=1`);
|
|
if (explicit) ffArgs.push('-metadata', 'ITUNESADVISORY=1');
|
|
if (lyricsData?.plain) ffArgs.push('-metadata', `lyrics=${lyricsData.plain}`);
|
|
if (hasCover) {
|
|
ffArgs.push(
|
|
'-metadata:s:v', 'title=Album cover',
|
|
'-metadata:s:v', 'comment=Cover (front)',
|
|
);
|
|
}
|
|
ffArgs.push(tmpFile, '-y');
|
|
|
|
const ff = spawn('ffmpeg', ffArgs);
|
|
let ffErr = '';
|
|
ff.stderr.on('data', d => ffErr += d.toString());
|
|
ff.on('error', () => resolve());
|
|
ff.on('close', code => {
|
|
if (code === 0 && fs.existsSync(tmpFile)) {
|
|
fs.renameSync(tmpFile, audioFile);
|
|
if (hasCover) try { fs.unlinkSync(coverPath); } catch {}
|
|
const base = path.basename(audioFile, path.extname(audioFile));
|
|
for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) {
|
|
try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {}
|
|
}
|
|
log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`);
|
|
} else {
|
|
try { fs.unlinkSync(tmpFile); } catch {}
|
|
log(jobId, `⚠️ ffmpeg code ${code}`);
|
|
if (ffErr) log(jobId, ffErr.slice(-200));
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
|
|
return { success: true, dir };
|
|
}
|
|
|
|
module.exports = { downloadTrack, embedMetadata };
|