Files
NovaSonic/backend/lib/ytdlp.js
T
HoneyandClaude Sonnet 4.6 1812500523 fix: YTM search for Japanese tracks - channel filter + album query context
- Skip channel filter when artistFirstWord is non-ASCII (Japanese kanji/kana):
  YouTube channels are named in romaji/English, not kanji — the filter was
  rejecting all valid results and causing 6+ extra cascade steps
- Add albumQuerySuffix: when title is Japanese (no keyword match) OR has only
  one romaji keyword, include the album's first two English words in the search
  query so YTM ranks the official OST/album version above fan covers

Fixes: 照井順政 channel filter rejecting all YTM results
Fixes: "Souiunatsu" matching covers with romaji subtitle instead of official OST

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-25 16:12:27 +02:00

392 lines
19 KiB
JavaScript

const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');
const CONFIG = require('./config');
const { log } = require('./jobs');
const {
sanitize, navidromeArtist,
downloadArtwork, findExistingTrack,
} = require('./metadata');
const { romajiForStorage } = require('./romaji');
function buildTitleKeywords(title) {
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
const lower = (title || '').toLowerCase();
const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim();
const words = strippedTitle
.replace(/[^\w\s]/g, ' ')
.split(/\s+/)
.filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase()));
const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2);
const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w;
const result = top2.map(toPattern);
const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null;
if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier);
return result;
}
function buildVariantFilter(title) {
const t = (title || '').toLowerCase();
// !~= (?i) = case-insensitive regex "does not match". Use \b word boundaries so that
// partial matches like "nightcoreals" containing "nightcore" don't falsely reject results.
const RULES = [
{ key: 'sped', expr: 'title !~= (?i)\\bsped\\b' },
{ key: 'speedup', expr: 'title !~= (?i)\\bspeed.{0,2}up\\b' },
{ key: 'slowed', expr: 'title !~= (?i)\\bslowed\\b' },
{ key: 'reverb', expr: 'title !~= (?i)\\breverb\\b' },
{ key: 'nightcore', expr: 'title !~= (?i)\\bnightcore\\b' },
{ key: 'karaoke', expr: 'title !~= (?i)\\bkaraoke\\b' },
{ key: 'parody', expr: 'title !~= (?i)\\bparody\\b' },
{ key: 'remix', expr: 'title !~= (?i)\\bremix\\b' },
{ key: 'cover', expr: 'title !~= (?i)\\bcover\\b' },
{ key: 'acoustic', expr: 'title !~= (?i)\\bacoustic\\b' },
{ key: 'instrumental', expr: 'title !~= (?i)\\binstrumental\\b' },
];
const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr);
return parts.length ? parts.join(' & ') : '';
}
async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs, sourceUrl }) {
const mainArtist = albumArtist || artist;
// Folder / filename: romaji of the resolved artist (iTunes name if available — already in English,
// otherwise kuromoji). rawArtist (original Japanese) is only used for tags + YTM search.
const mainArtistRomaji = romajiForStorage(mainArtist) || mainArtist;
const artistRomaji = romajiForStorage(artist) || artist;
const titleRomaji = romajiForStorage(title) || title;
const albumRomaji = album ? (romajiForStorage(album) || album) : null;
const safeArtist = sanitize(mainArtistRomaji.split(/[,;&]/)[0].trim());
const safeAlbum = sanitize(albumRomaji || 'Singles');
const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum);
fs.mkdirSync(dir, { recursive: true });
const safeTitle = sanitize(titleRomaji);
const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : '';
// Filename format: "{01 - }Artist1, Artist2 - Title.ext"
// Use track artist (not albumArtist) so all credited artists appear in the filename
const safeArtistForFile = sanitize(artistRomaji.replace(/;/g, ','));
const fileBase = `${trackPrefix}${safeArtistForFile} - ${safeTitle}`;
const existing = findExistingTrack(dir, trackPrefix, safeTitle, safeArtistForFile);
if (existing) {
log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`);
return { success: true, dir, skipped: true };
}
const outputTemplate = path.join(dir, `${fileBase}.%(ext)s`);
// For YTM search: use the original (raw) artist when available — the YTM channel is registered
// under the Japanese/original name, not the iTunes-resolved English name.
// rawArtist is undefined for non-Japanese tracks, so fall back to mainArtist.
const ytmArtist = (rawArtist || mainArtist).replace(/\p{S}/gu, '').trim();
const searchArtist = ytmArtist;
const query = `${ytmArtist} - ${title}${albumQuerySuffix}`;
const baseArgs = [
'--extract-audio',
'--audio-format', CONFIG.audioFmt,
'--audio-quality', CONFIG.audioQuality,
'-o', outputTemplate,
'--no-playlist',
'--no-warnings',
'--newline',
];
const spawnYtdlp = (args) => new Promise((resolve, reject) => {
const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
let stderr = '', lastPct = '';
proc.stdout.on('data', d => {
d.toString().split('\n').filter(l => l.trim()).forEach(line => {
if (line.includes('%') && line === lastPct) return;
if (line.includes('%')) lastPct = line;
log(jobId, line.trim());
});
});
proc.stderr.on('data', d => {
stderr += d.toString();
d.toString().split('\n')
.filter(l => l.trim() && !l.startsWith('[debug]'))
.forEach(l => log(jobId, l.trim()));
});
proc.on('error', reject);
proc.on('close', code =>
(code === 0 || code === 101) ? resolve() : reject(new Error(`yt-dlp code ${code}: ${stderr.slice(-300)}`))
);
});
const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D';
// Use original (raw) artist for YTM URL — better ranking since channel is in original language
const ytmQuery = `${rawArtist || mainArtist} - ${title}${albumQuerySuffix}`;
const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(ytmQuery)}&params=${YTM_SONGS_PARAM}`;
const expectedAudio = path.join(dir, `${fileBase}.${CONFIG.audioFmt}`);
const hasAudio = () => fs.existsSync(expectedAudio);
// Direct URL download (e.g. SoundCloud) — skip the ytsearch cascade entirely
if (sourceUrl) {
log(jobId, `🔗 Direkter Download: ${sourceUrl}`);
await spawnYtdlp([sourceUrl, '--no-playlist', ...baseArgs]);
const audioFile = hasAudio() ? expectedAudio : null;
if (!audioFile) { log(jobId, '⚠️ Download fehlgeschlagen'); return { success: true, dir }; }
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
}
// Fetch YTM search result URLs as a flat list (no channel expansion) and return only
// actual video URLs (watch?v=…). This prevents yt-dlp from following an artist channel
// result and treating its uploads as the search result set.
//
// yt-dlp expands "url_transparent" artist channel entries even under --flat-playlist,
// printing the channel's video URLs instead of the channel URL. We detect this by
// watching for [youtube:tab] in output (which only fires during channel/tab extraction)
// and return [] so the caller falls through to the ytsearch fallback.
const getYtmVideoUrls = (maxItems) => new Promise(resolve => {
const urls = [];
let channelExpanded = false;
const checkExpansion = (text) => {
if (text.includes('[youtube:tab]') || text.includes('/browse/UC')) channelExpanded = true;
};
const proc = spawn(CONFIG.ytdlpBin, [
ytmUrl,
'--flat-playlist',
'--playlist-items', `1-${maxItems}`,
'--print', 'webpage_url',
'--no-warnings',
], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
proc.stdout.on('data', d => {
const text = d.toString();
checkExpansion(text);
if (!channelExpanded) {
text.split('\n').forEach(line => {
const u = line.trim();
if (u.includes('watch?v=')) urls.push(u);
});
}
});
proc.stderr.on('data', d => checkExpansion(d.toString()));
proc.on('error', () => resolve([]));
proc.on('close', () => resolve(channelExpanded ? [] : urls));
});
const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const JP_CHARS = /[ぁ-んァ-ン一-龯]/;
const variantFilter = buildVariantFilter(title);
const titleKws = buildTitleKeywords(title);
const variantPart = variantFilter ? ` & ${variantFilter}` : '';
const titleKwPart = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : '';
// Soft channel check: first word of original artist name used in early cascade steps to avoid
// downloading a same-title song by a different artist (e.g. wrong "Moonlight" for N!GHT).
// Skip when artistFirstWord is non-ASCII (Japanese kanji/kana): YouTube channel names are
// almost never in kanji, so the filter would reject every valid result.
const artistFirstWord = (rawArtist || mainArtist).split(/[\s,/]+/)[0];
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_CHARS.test(artistFirstWord))
? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';
// When the title is Japanese (keywords stripped to nothing) OR when we have an album that
// provides better search context, include the album's first two words in the search query.
// This helps YTM rank the official OST/album track above fan covers and romaji-subtitle uploads.
// Only use album words that are ASCII (romaji/English) — Japanese album words don't help YTM ranking.
const hasJpTitle = JP_CHARS.test(title);
const albumWords = (album && album !== 'Singles' && album !== 'Unknown')
? album.split(/\s+/).filter(w => /^[A-Za-z0-9]/.test(w)).slice(0, 2).join(' ')
: '';
const albumQuerySuffix = (hasJpTitle || (albumWords && titleKws.length <= 1)) && albumWords
? ' ' + albumWords : '';
const mf = (dur, variant, strict = false) => {
const durPart = dur ? `duration > ${dur.min} & duration < ${dur.max} & ` : '';
return ['--match-filter',
`${durPart}!is_live${titleKwPart}${strict ? artistChannelPart : ''}${variant ? variantPart : ''}`];
};
const PI_FILTERED = ['--playlist-items', '1-5', '--max-downloads', '1'];
const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
const PI_UNFILTERED = ['--playlist-items', '1'];
// Fetch YTM URLs once upfront so every cascade step filters the SAME result set.
// Re-fetching produced different orderings, causing the correct video (found in step 1
// but failing the channel filter) to disappear by step 3 and a wrong one to slip through.
const ytmUrls = await getYtmVideoUrls(10);
const tryYtm = async (n, mfArgs) => {
const urls = n ? ytmUrls.slice(0, n) : ytmUrls;
if (!urls.length) return;
await spawnYtdlp([...urls, '--max-downloads', '1', ...mfArgs, ...baseArgs]);
};
if (durationMs) {
const durationSec = Math.round(durationMs / 1000);
const margin = Math.max(15, Math.round(durationSec * 0.15));
const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };
log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
// 1. YTM top-5 — strict: duration + title + variant + channel check
await tryYtm(5, mf(dur, true, true));
if (!hasAudio()) {
// 2. YT top-5 — strict: same filters including channel check
log(jobId, '⚠️ Kein YTM-Treffer — Fallback zu YouTube…');
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true, true), ...baseArgs]);
}
if (!hasAudio()) {
// 3. YTM all-10 — no channel check (label channels, etc.)
log(jobId, '⚠️ Erweitere Suche auf top 10 mit Dauerbeschränkung…');
await tryYtm(0, mf(dur, true));
}
if (!hasAudio()) {
// 4. YT top-10 — no channel check
await spawnYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
}
if (!hasAudio()) {
// 5. YTM all — no duration check
log(jobId, '⚠️ Kein Treffer im Dauerbereich — suche ohne Dauerbeschränkung…');
await tryYtm(0, mf(null, true));
}
if (!hasAudio()) {
// 6. YT top-5 — no duration check
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
}
if (!hasAudio()) {
// 7. YTM all — title only (no variant filter)
log(jobId, '⚠️ Letzter gefilterter Versuch…');
await tryYtm(0, mf(null, false));
}
if (!hasAudio()) {
// 8. YT top-5 — title only
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
}
if (!hasAudio()) {
// 9. YTM — no filter at all, take first video result
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
await tryYtm(3, []);
}
if (!hasAudio()) {
// 10. YT — no filter
await spawnYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
}
} else {
log(jobId, `🔍 Suche YT Music: "${query}"`);
// 1. YTM top-5 — strict: title + variant + channel check
await tryYtm(5, mf(null, true, true));
if (!hasAudio()) {
// 2. YT top-5 — strict
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true, true), ...baseArgs]);
}
if (!hasAudio()) {
// 3. YTM all-10 — no channel check (catches label/distributor channels)
await tryYtm(0, mf(null, true));
}
if (!hasAudio()) {
// 4. YT top-5 — no channel check
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
}
if (!hasAudio()) {
// 5. YTM all — title only
log(jobId, '⚠️ Letzter gefilterter Versuch…');
await tryYtm(0, mf(null, false));
}
if (!hasAudio()) {
// 6. YT top-5 — title only
await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, false), ...baseArgs]);
}
if (!hasAudio()) {
// 7. YTM — first video result, no filter
log(jobId, '⚠️ Letzter Versuch: keine Filter…');
await tryYtm(3, []);
}
if (!hasAudio()) {
// 8. YT — no filter
await spawnYtdlp([`ytsearch1:${query}`, ...PI_UNFILTERED, ...baseArgs]);
}
}
const audioFile = hasAudio() ? expectedAudio : null;
if (!audioFile) {
log(jobId, '⚠️ Keine Audio-Datei gefunden');
return { success: true, dir };
}
return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title });
}
// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
async function embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title }) {
const coverPath = await downloadArtwork(artworkUrl, dir);
const hasCover = !!(coverPath && fs.existsSync(coverPath));
log(jobId, `🏷 Metadaten + Cover → ${path.basename(audioFile)}`);
const fileExt = path.extname(audioFile);
const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt;
await new Promise(resolve => {
const ffArgs = ['-i', audioFile];
if (hasCover) ffArgs.push('-i', coverPath);
ffArgs.push('-map', '0:a');
if (hasCover) ffArgs.push('-map', '1:v');
// Main tags: store original text (Japanese or whatever was passed in).
// rawArtist = original Spotify/source artist (may be Japanese); artist = iTunes-resolved (may be English).
// Use rawArtist for the stored tag so Apple Music / other apps see the authentic original.
const tagArtist = navidromeArtist(rawArtist || artist);
const tagAlbumArtist = (rawArtist || mainArtist).replace(/\s+&\s+/g, ', ');
// Sort tags: romaji for display / sorting in apps that support them (TITLESORT, ARTISTSORT, etc.)
// artist is already the iTunes-resolved English name when available — use directly (more accurate
// than kuromoji). For non-iTunes tracks romajiForStorage acts as kuromoji fallback.
const sortTitle = romajiForStorage(title) || title;
const sortArtist = romajiForStorage(navidromeArtist(artist)) || navidromeArtist(artist);
const sortAlbumArtist = romajiForStorage(mainArtist.replace(/\s+&\s+/g, ', ')) || mainArtist.replace(/\s+&\s+/g, ', ');
const sortAlbum = album ? (romajiForStorage(album) || album) : null;
ffArgs.push(
'-c', 'copy',
'-map_metadata', '-1',
'-id3v2_version', '3',
'-metadata', `title=${title}`,
'-metadata', `artist=${tagArtist}`,
'-metadata', `album_artist=${tagAlbumArtist}`,
);
if (album) ffArgs.push('-metadata', `album=${album}`);
// Sort tags — only write when they differ from main (no-op for non-Japanese tracks)
if (sortTitle !== title) ffArgs.push('-metadata', `TITLESORT=${sortTitle}`);
if (sortArtist !== tagArtist) ffArgs.push('-metadata', `ARTISTSORT=${sortArtist}`);
if (sortAlbumArtist !== tagAlbumArtist) ffArgs.push('-metadata', `ALBUMARTISTSORT=${sortAlbumArtist}`);
if (sortAlbum && sortAlbum !== album) ffArgs.push('-metadata', `ALBUMSORT=${sortAlbum}`);
if (year) ffArgs.push('-metadata', `date=${String(year)}`);
if (trackNum) ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`);
if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`);
if (genre) ffArgs.push('-metadata', `genre=${genre}`);
if (discNum) ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`);
if (compilation) ffArgs.push('-metadata', `compilation=1`);
if (explicit) ffArgs.push('-metadata', 'ITUNESADVISORY=1');
if (hasCover) {
ffArgs.push(
'-metadata:s:v', 'title=Album cover',
'-metadata:s:v', 'comment=Cover (front)',
);
}
ffArgs.push(tmpFile, '-y');
const ff = spawn('ffmpeg', ffArgs);
let ffErr = '';
ff.stderr.on('data', d => ffErr += d.toString());
ff.on('error', () => resolve());
ff.on('close', code => {
if (code === 0 && fs.existsSync(tmpFile)) {
fs.renameSync(tmpFile, audioFile);
if (hasCover) try { fs.unlinkSync(coverPath); } catch {}
const base = path.basename(audioFile, path.extname(audioFile));
for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) {
try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {}
}
log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`);
} else {
try { fs.unlinkSync(tmpFile); } catch {}
log(jobId, `⚠️ ffmpeg code ${code}`);
if (ffErr) log(jobId, ffErr.slice(-200));
}
resolve();
});
});
return { success: true, dir };
}
module.exports = { downloadTrack, embedMetadata };