fix: pre-filter YTM search results to skip artist channel expansion
--match-filter 'webpage_url !*= /browse/' was applied to individual videos AFTER yt-dlp expanded an artist channel into its uploads playlist — by then each video had a /watch?v= URL, so the filter never fired. New approach: run yt-dlp --flat-playlist --print webpage_url first to get the raw search result URLs, filter /browse/ entries in Node.js, then pass only the actual watch?v= video URLs to yt-dlp for download with --max-downloads 1. The channel is never expanded; we get the real song from the search results. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
ac8d26bf9d
commit
7aad4989f3
1 file changed
+62
-23
+62
-23
@@ -8,15 +8,6 @@ const {
|
|||||||
downloadArtwork, fetchLyrics, findExistingTrack,
|
downloadArtwork, fetchLyrics, findExistingTrack,
|
||||||
} = require('./metadata');
|
} = require('./metadata');
|
||||||
|
|
||||||
// Extract a plain word from an artist name for yt-dlp --match-filter uploader check.
|
|
||||||
function artistFilterWord(name) {
|
|
||||||
const first = (name || '').split(/[,/•]/)[0].trim();
|
|
||||||
const runs = first.match(/\w+/g) || [];
|
|
||||||
const best = runs.filter(r => r.length >= 3).sort((a, b) => b.length - a.length)[0];
|
|
||||||
return best || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract distinctive words from a track title for yt-dlp title ~= (?i) filters.
|
|
||||||
function buildTitleKeywords(title) {
|
function buildTitleKeywords(title) {
|
||||||
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
|
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
|
||||||
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
|
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
|
||||||
@@ -35,7 +26,6 @@ function buildTitleKeywords(title) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build yt-dlp --match-filter conditions to reject variant versions.
|
|
||||||
function buildVariantFilter(title) {
|
function buildVariantFilter(title) {
|
||||||
const t = (title || '').toLowerCase();
|
const t = (title || '').toLowerCase();
|
||||||
const RULES = [
|
const RULES = [
|
||||||
@@ -83,11 +73,7 @@ async function downloadTrack({ jobId, artist, albumArtist, title, album, year, t
|
|||||||
'--no-playlist',
|
'--no-playlist',
|
||||||
'--no-warnings',
|
'--no-warnings',
|
||||||
'--newline',
|
'--newline',
|
||||||
'--match-filter', 'webpage_url !*= /browse/',
|
|
||||||
];
|
];
|
||||||
const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1'];
|
|
||||||
const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
|
|
||||||
const PI_UNFILTERED = ['--playlist-items', '1'];
|
|
||||||
|
|
||||||
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
|
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
|
||||||
const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
||||||
@@ -117,6 +103,39 @@ async function downloadTrack({ jobId, artist, albumArtist, title, album, year, t
|
|||||||
const hasAudio = () => fs.existsSync(expectedAudio);
|
const hasAudio = () => fs.existsSync(expectedAudio);
|
||||||
const tryYtdlp = async (args) => { await spawnYtdlp(args); };
|
const tryYtdlp = async (args) => { await spawnYtdlp(args); };
|
||||||
|
|
||||||
|
// Fetch YTM search result URLs as a flat list (no channel expansion) and return only
|
||||||
|
// actual video URLs (watch?v=…). This prevents yt-dlp from following an artist channel
|
||||||
|
// result and treating its uploads as the search result set.
|
||||||
|
const getYtmVideoUrls = (maxItems) => new Promise(resolve => {
|
||||||
|
const urls = [];
|
||||||
|
const proc = spawn(CONFIG.ytdlpBin, [
|
||||||
|
ytmUrl,
|
||||||
|
'--flat-playlist',
|
||||||
|
'--playlist-items', `1-${maxItems}`,
|
||||||
|
'--print', 'webpage_url',
|
||||||
|
'--no-warnings',
|
||||||
|
], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
|
||||||
|
proc.stdout.on('data', d => {
|
||||||
|
d.toString().split('\n').forEach(line => {
|
||||||
|
const u = line.trim();
|
||||||
|
// Only accept video URLs — skip /browse/ (artist channel) and /playlist? entries
|
||||||
|
if (u.includes('watch?v=')) urls.push(u);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
proc.stderr.on('data', () => {});
|
||||||
|
proc.on('error', () => resolve([]));
|
||||||
|
proc.on('close', () => resolve(urls));
|
||||||
|
});
|
||||||
|
|
||||||
|
// Search YTM for video URLs (pre-filtered), then pass them all at once to yt-dlp.
|
||||||
|
// --max-downloads 1 makes yt-dlp stop after the first video that passes mfArgs.
|
||||||
|
// Filtered/skipped videos do NOT count toward --max-downloads, so the cascade works.
|
||||||
|
const tryYtmSearch = async (maxItems, mfArgs) => {
|
||||||
|
const urls = await getYtmVideoUrls(maxItems);
|
||||||
|
if (!urls.length) return;
|
||||||
|
await tryYtdlp([...urls, '--max-downloads', '1', ...mfArgs, ...baseArgs]);
|
||||||
|
};
|
||||||
|
|
||||||
const variantFilter = buildVariantFilter(title);
|
const variantFilter = buildVariantFilter(title);
|
||||||
const titleKws = buildTitleKeywords(title);
|
const titleKws = buildTitleKeywords(title);
|
||||||
const variantPart = variantFilter ? ` & ${variantFilter}` : '';
|
const variantPart = variantFilter ? ` & ${variantFilter}` : '';
|
||||||
@@ -128,63 +147,83 @@ async function downloadTrack({ jobId, artist, albumArtist, title, album, year, t
|
|||||||
`${durPart}!is_live${titleKwPart}${variant ? variantPart : ''}`];
|
`${durPart}!is_live${titleKwPart}${variant ? variantPart : ''}`];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1'];
|
||||||
|
const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
|
||||||
|
const PI_UNFILTERED = ['--playlist-items', '1'];
|
||||||
|
|
||||||
if (durationMs) {
|
if (durationMs) {
|
||||||
const durationSec = Math.round(durationMs / 1000);
|
const durationSec = Math.round(durationMs / 1000);
|
||||||
const margin = Math.max(15, Math.round(durationSec * 0.15));
|
const margin = Math.max(15, Math.round(durationSec * 0.15));
|
||||||
const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };
|
const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };
|
||||||
|
|
||||||
log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
|
log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]);
|
// 1. YTM top-5 — pre-filtered URLs (no channel expansion), duration + title + variant filter
|
||||||
|
await tryYtmSearch(5, mf(dur, true));
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 2. YT top-5 — ytsearch always returns videos (no channels), safe to use directly
|
||||||
log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…');
|
log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…');
|
||||||
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]);
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 3. YTM top-10 — wider net
|
||||||
log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…');
|
log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
|
await tryYtmSearch(10, mf(dur, true));
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 4. YT top-10
|
||||||
await tryYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
|
await tryYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 5. YTM top-5 — no duration check
|
||||||
log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…');
|
log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
await tryYtmSearch(5, mf(null, true));
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 6. YT top-5 — no duration check
|
||||||
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 7. YTM top-5 — title only (no variant filter)
|
||||||
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
await tryYtmSearch(5, mf(null, false));
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 8. YT top-5 — title only
|
||||||
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 9. YTM — no filter at all, take first video result
|
||||||
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]);
|
await tryYtmSearch(3, []);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 10. YT — no filter
|
||||||
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log(jobId, `🔍 Suche YT Music: "${query}"`);
|
log(jobId, `🔍 Suche YT Music: "${query}"`);
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
// 1. YTM top-5 — pre-filtered, title + variant
|
||||||
|
await tryYtmSearch(5, mf(null, true));
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 2. YT top-5
|
||||||
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 3. YTM top-5 — title only
|
||||||
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
log(jobId, '⚠️ Letzter gefilterter Versuch…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
await tryYtmSearch(5, mf(null, false));
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 4. YT top-5 — title only
|
||||||
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
await tryYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 5. YTM — first video result, no filter
|
||||||
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
|
||||||
await tryYtdlp([ytmUrl, ...PI_UNFILTERED, ...baseArgs]);
|
await tryYtmSearch(3, []);
|
||||||
}
|
}
|
||||||
if (!hasAudio()) {
|
if (!hasAudio()) {
|
||||||
|
// 6. YT — no filter
|
||||||
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
await tryYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -195,7 +234,7 @@ async function downloadTrack({ jobId, artist, albumArtist, title, album, year, t
|
|||||||
return { success: true, dir };
|
return { success: true, dir };
|
||||||
}
|
}
|
||||||
|
|
||||||
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title, lyricsArtist: artist });
|
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
|
// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
|
||||||
|
|||||||
Reference in new issue
Block a user