fix: repair Japanese song download pipeline
Five separate root causes fixed: 1. buildTitleKeywords (ytdlp.js): For Japanese titles \w strips all kana/kanji leaving no keywords. Now uses the first whitespace- delimited segment as a literal regex keyword so --match-filter title ~= requires the YTM video to contain the Japanese text. 2. Tokenizer wait (ytdlp.js): If kuromoji hasn't finished loading when a Japanese download starts, romajiForStorage returns the original Japanese text unchanged — producing Japanese folder/filenames and skipping sort tags. Now waits up to 30 s for the tokenizer. 3. titleOnlyMatch guard (metadata.js): iTunes titleOnlyMatch (no artist check) was used as a fallback for all searches. For Japanese titles/artists this matched wrong US songs with identical titles (common in anime OSTs). Now titleOnlyMatch is skipped when the search title or artist contains Japanese characters. 4. squid.js rawArtist: rawArtist was not in squid's signature and never forwarded to embedMetadata, so Qobuz-downloaded J-Pop tracks stored the iTunes-resolved English name as artist instead of the Japanese original. Fixed by accepting and forwarding rawArtist. 5. squid.js romaji paths: Folder and filename used raw artist/title with no romaji conversion, creating Japanese-named paths inconsistent with the yt-dlp path. Now uses romajiForStorage same as ytdlp.js. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
52451f5086
commit
63ff8920bf
3 files changed
+45
-10
No files matched your search
+29
-4
@@ -7,9 +7,20 @@ const {
|
||||
sanitize, navidromeArtist,
|
||||
downloadArtwork, findExistingTrack,
|
||||
} = require('./metadata');
|
||||
const { romajiForStorage } = require('./romaji');
|
||||
const { romajiForStorage, isTokenizerReady } = require('./romaji');
|
||||
|
||||
// Matches hiragana, katakana, CJK — used to detect Japanese content
|
||||
const JP_RE = /[ぁ-んァ-ン一-龯]/;
|
||||
|
||||
function buildTitleKeywords(title) {
|
||||
// For Japanese titles: \w strips all kana/kanji, leaving nothing useful.
|
||||
// Instead use the first whitespace-delimited segment as a literal keyword so
|
||||
// the YTM --match-filter requires the video title to contain it.
|
||||
if (JP_RE.test(title)) {
|
||||
const firstWord = (title.split(/[\s ]/)[0] || '').slice(0, 12);
|
||||
const escaped = firstWord.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
return escaped ? [escaped] : [];
|
||||
}
|
||||
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
|
||||
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
|
||||
const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
|
||||
@@ -51,6 +62,21 @@ function buildVariantFilter(title) {
|
||||
async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs, sourceUrl }) {
|
||||
const mainArtist = albumArtist || artist;
|
||||
|
||||
// If the title or artist contains Japanese characters, wait for kuromoji to finish loading.
|
||||
// Without it romajiForStorage returns the original text unchanged, producing Japanese
|
||||
// folder/filenames and preventing sort tags from being written.
|
||||
if (JP_RE.test(title) || JP_RE.test(rawArtist || artist || '')) {
|
||||
if (!isTokenizerReady()) {
|
||||
log(jobId, '⏳ Warte auf kuromoji-Tokenizer…');
|
||||
await new Promise(resolve => {
|
||||
const t0 = Date.now();
|
||||
const iv = setInterval(() => {
|
||||
if (isTokenizerReady() || Date.now() - t0 > 30000) { clearInterval(iv); resolve(); }
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Folder / filename: romaji of the resolved artist (iTunes name if available — already in English,
|
||||
// otherwise kuromoji). rawArtist (original Japanese) is only used for tags + YTM search.
|
||||
const mainArtistRomaji = romajiForStorage(mainArtist) || mainArtist;
|
||||
@@ -83,7 +109,6 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
|
||||
const searchArtist = ytmArtist;
|
||||
|
||||
const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const JP_CHARS = /[ぁ-んァ-ン一-龯]/;
|
||||
|
||||
const variantFilter = buildVariantFilter(title);
|
||||
const titleKws = buildTitleKeywords(title);
|
||||
@@ -95,14 +120,14 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
|
||||
// Skip when artistFirstWord is non-ASCII (Japanese kanji/kana): YouTube channel names are
|
||||
// almost never in kanji, so the filter would reject every valid result.
|
||||
const artistFirstWord = (rawArtist || mainArtist).split(/[\s,/]+/)[0];
|
||||
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_CHARS.test(artistFirstWord))
|
||||
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_RE.test(artistFirstWord))
|
||||
? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';
|
||||
|
||||
// When the title is Japanese (keywords stripped to nothing) OR when we have an album that
|
||||
// provides better search context, include the album's first two words in the search query.
|
||||
// This helps YTM rank the official OST/album track above fan covers and romaji-subtitle uploads.
|
||||
// Only use album words that are ASCII (romaji/English) — Japanese album words don't help YTM ranking.
|
||||
const hasJpTitle = JP_CHARS.test(title);
|
||||
const hasJpTitle = JP_RE.test(title);
|
||||
const albumWords = (album && album !== 'Singles' && album !== 'Unknown')
|
||||
? album.split(/\s+/).filter(w => /^[A-Za-z0-9]/.test(w)).slice(0, 2).join(' ')
|
||||
: '';
|
||||
|
||||
Reference in new issue
Block a user