fix: repair Japanese song download pipeline

Five separate root causes fixed:

1. buildTitleKeywords (ytdlp.js): For Japanese titles \w strips all
   kana/kanji leaving no keywords. Now uses the first whitespace-
   delimited segment as a literal regex keyword so --match-filter
   title ~= requires the YTM video to contain the Japanese text.

2. Tokenizer wait (ytdlp.js): If kuromoji hasn't finished loading when
   a Japanese download starts, romajiForStorage returns the original
   Japanese text unchanged — producing Japanese folder/filenames and
   skipping sort tags. Now waits up to 30 s for the tokenizer.

3. titleOnlyMatch guard (metadata.js): iTunes titleOnlyMatch (no artist
   check) was used as a fallback for all searches. For Japanese
   titles/artists this matched wrong US songs with identical titles
   (common in anime OSTs). Now titleOnlyMatch is skipped when the
   search title or artist contains Japanese characters.

4. squid.js rawArtist: rawArtist was not in squid's signature and
   never forwarded to embedMetadata, so Qobuz-downloaded J-Pop tracks
   stored the iTunes-resolved English name as artist instead of the
   Japanese original. Fixed by accepting and forwarding rawArtist.

5. squid.js romaji paths: Folder and filename used raw artist/title
   with no romaji conversion, creating Japanese-named paths
   inconsistent with the yt-dlp path. Now uses romajiForStorage same
   as ytdlp.js.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
HoneyandClaude Sonnet 4.6 committed 2026-05-29 01:40:46 +02:00
1 parent 52451f5086
commit 63ff8920bf
3 files changed
+45 -10

No files matched your search

+29 -4
View File
@@ -7,9 +7,20 @@ const {
sanitize, navidromeArtist,
downloadArtwork, findExistingTrack,
} = require('./metadata');
const { romajiForStorage } = require('./romaji');
const { romajiForStorage, isTokenizerReady } = require('./romaji');
// Matches hiragana, katakana, CJK — used to detect Japanese content
const JP_RE = /[ぁ-んァ-ン一-龯]/;
function buildTitleKeywords(title) {
// For Japanese titles: \w strips all kana/kanji, leaving nothing useful.
// Instead use the first whitespace-delimited segment as a literal keyword so
// the YTM --match-filter requires the video title to contain it.
if (JP_RE.test(title)) {
const firstWord = (title.split(/[\s ]/)[0] || '').slice(0, 12);
const escaped = firstWord.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
return escaped ? [escaped] : [];
}
const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with']);
const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
@@ -51,6 +62,21 @@ function buildVariantFilter(title) {
async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs, sourceUrl }) {
const mainArtist = albumArtist || artist;
// If the title or artist contains Japanese characters, wait for kuromoji to finish loading.
// Without it romajiForStorage returns the original text unchanged, producing Japanese
// folder/filenames and preventing sort tags from being written.
if (JP_RE.test(title) || JP_RE.test(rawArtist || artist || '')) {
if (!isTokenizerReady()) {
log(jobId, '⏳ Warte auf kuromoji-Tokenizer…');
await new Promise(resolve => {
const t0 = Date.now();
const iv = setInterval(() => {
if (isTokenizerReady() || Date.now() - t0 > 30000) { clearInterval(iv); resolve(); }
}, 300);
});
}
}
// Folder / filename: romaji of the resolved artist (iTunes name if available — already in English,
// otherwise kuromoji). rawArtist (original Japanese) is only used for tags + YTM search.
const mainArtistRomaji = romajiForStorage(mainArtist) || mainArtist;
@@ -83,7 +109,6 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
const searchArtist = ytmArtist;
const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const JP_CHARS = /[ぁ-んァ-ン一-龯]/;
const variantFilter = buildVariantFilter(title);
const titleKws = buildTitleKeywords(title);
@@ -95,14 +120,14 @@ async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, alb
// Skip when artistFirstWord is non-ASCII (Japanese kanji/kana): YouTube channel names are
// almost never in kanji, so the filter would reject every valid result.
const artistFirstWord = (rawArtist || mainArtist).split(/[\s,/]+/)[0];
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_CHARS.test(artistFirstWord))
const artistChannelPart = (artistFirstWord.length >= 3 && !JP_RE.test(artistFirstWord))
? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';
// When the title is Japanese (keywords stripped to nothing) OR when we have an album that
// provides better search context, include the album's first two words in the search query.
// This helps YTM rank the official OST/album track above fan covers and romaji-subtitle uploads.
// Only use album words that are ASCII (romaji/English) — Japanese album words don't help YTM ranking.
const hasJpTitle = JP_CHARS.test(title);
const hasJpTitle = JP_RE.test(title);
const albumWords = (album && album !== 'Singles' && album !== 'Unknown')
? album.split(/\s+/).filter(w => /^[A-Za-z0-9]/.test(w)).slice(0, 2).join(' ')
: '';