NovaSonic/backend/lib/ytdlp.js

const { spawn } = require('child_process');
const path      = require('path');
const fs        = require('fs');
const CONFIG    = require('./config');
const { log }   = require('./jobs');
const {
  sanitize, navidromeArtist,
  downloadArtwork, findExistingTrack,
} = require('./metadata');
const { romajiForStorage, isTokenizerReady } = require('./romaji');

// Matches hiragana, katakana, CJK — used to detect Japanese content
const JP_RE = /[ぁ-んァ-ン一-龯]/;

function buildTitleKeywords(title) {
  // For Japanese titles: \w strips all kana/kanji, leaving nothing useful.
  // Instead use the first whitespace-delimited segment as a literal keyword so
  // the YTM --match-filter requires the video title to contain it.
  if (JP_RE.test(title)) {
    const firstWord = (title.split(/[\s　]/)[0] || '').slice(0, 12);
    const escaped   = firstWord.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
    return escaped ? [escaped] : [];
  }
  const VARIANT_WORDS = new Set(['sped','speedup','slowed','reverb','nightcore','karaoke','parody','remix','cover','acoustic','instrumental']);
  const STOPWORDS = new Set(['the','a','an','and','or','of','in','on','at','to','for','feat','ft','with','single','ep','album','remaster','remastered','deluxe','edition','version','live','bonus']);
  const MIX_QUALIFIERS = ['radio', 'extended', 'club', 'vip', 'original', 'edit'];
  const lower = (title || '').toLowerCase();
  const strippedTitle = (title || '').replace(/\([^)]*\)/g, '').trim();
  const words = strippedTitle
    .replace(/[^\w\s]/g, ' ')
    .split(/\s+/)
    .filter(w => w.length >= 3 && !VARIANT_WORDS.has(w.toLowerCase()) && !STOPWORDS.has(w.toLowerCase()));
  const top2 = words.sort((a, b) => b.length - a.length).slice(0, 2);
  const toPattern = w => (w.length >= 4 && w.endsWith('s')) ? w + '?' : w;
  const result = top2.map(toPattern);
  const qualifier = lower.includes('mix') ? MIX_QUALIFIERS.find(q => lower.includes(q)) : null;
  if (qualifier && !top2.some(w => w.toLowerCase() === qualifier)) result.push(qualifier);
  return result;
}

function buildVariantFilter(title) {
  const t = (title || '').toLowerCase();
  // !~= (?i) = case-insensitive regex "does not match". Use \b word boundaries so that
  // partial matches like "nightcoreals" containing "nightcore" don't falsely reject results.
  const RULES = [
    { key: 'sped',         expr: 'title !~= (?i)\\bsped\\b' },
    { key: 'speedup',      expr: 'title !~= (?i)\\bspeed.{0,2}up\\b' },
    { key: 'slowed',       expr: 'title !~= (?i)\\bslowed\\b' },
    { key: 'reverb',       expr: 'title !~= (?i)\\breverb\\b' },
    { key: 'nightcore',    expr: 'title !~= (?i)\\bnightcore\\b' },
    { key: 'karaoke',      expr: 'title !~= (?i)\\bkaraoke\\b' },
    { key: 'parody',       expr: 'title !~= (?i)\\bparody\\b' },
    { key: 'remix',        expr: 'title !~= (?i)\\bremix\\b' },
    { key: 'cover',        expr: 'title !~= (?i)\\bcover\\b' },
    { key: 'acoustic',     expr: 'title !~= (?i)\\bacoustic\\b' },
    { key: 'instrumental', expr: 'title !~= (?i)\\binstrumental\\b' },
  ];
  const parts = RULES.filter(r => !t.includes(r.key)).map(r => r.expr);
  return parts.length ? parts.join(' & ') : '';
}

async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs, sourceUrl }) {
  const mainArtist = albumArtist || artist;

  // If the title or artist contains Japanese characters, wait for kuromoji to finish loading.
  // Without it romajiForStorage returns the original text unchanged, producing Japanese
  // folder/filenames and preventing sort tags from being written.
  if (JP_RE.test(title) || JP_RE.test(rawArtist || artist || '')) {
    if (!isTokenizerReady()) {
      log(jobId, '⏳ Warte auf kuromoji-Tokenizer…');
      await new Promise(resolve => {
        const t0 = Date.now();
        const iv = setInterval(() => {
          if (isTokenizerReady() || Date.now() - t0 > 30000) { clearInterval(iv); resolve(); }
        }, 300);
      });
    }
  }

  // Strip release-type suffixes Spotify appends to track names (e.g. "recognize - Single" → "recognize").
  // Only strip trailing "- word(s)" when those words are release-type qualifiers or bare years, not
  // real sub-titles (which tend to be longer phrases or start with "feat"/"ft").
  // Used for BOTH the filename ("01 - Artist - Title", no "- Single") and the YTM search query.
  const RELEASE_SUFFIX_RE = /\s*[-–]\s*(?:(?:official\s+)?(?:music\s+)?(?:single|ep|lp|album|ost|soundtrack|remaster(?:ed)?|deluxe(?:\s+edition)?|bonus\s+tracks?|b-?sides?|version|edit)\b.*|(?:20|19)\d{2})\s*$/i;
  const cleanTitle = title.replace(RELEASE_SUFFIX_RE, '').trim() || title;

  // Folder / filename: use original names (Japanese or whatever was passed in).
  // Romaji goes ONLY into sort tags (TITLESORT, ARTISTSORT, etc.) via embedMetadata.
  const safeArtist        = sanitize((mainArtist || '').split(/[,;&]/)[0].trim());
  const safeAlbum         = sanitize(album || 'Singles');
  const dir               = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum);
  fs.mkdirSync(dir, { recursive: true });

  const safeTitle          = sanitize(cleanTitle);
  const trackPrefix        = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : '';
  // Filename format: "{01 - }Artist1, Artist2 - Title.ext"
  const safeArtistForFile  = sanitize((artist || '').replace(/;/g, ','));
  const fileBase           = `${trackPrefix}${safeArtistForFile} - ${safeTitle}`;

  const existing = findExistingTrack(outputDir || CONFIG.musicPath, safeArtist, cleanTitle);
  if (existing) {
    log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`);
    return { success: true, dir, skipped: true };
  }

  const outputTemplate = path.join(dir, `${fileBase}.%(ext)s`);
  // For YTM search: use the original (raw) artist when available — the YTM channel is registered
  // under the Japanese/original name, not the iTunes-resolved English name.
  // rawArtist is undefined for non-Japanese tracks, so fall back to mainArtist.
  const ytmArtist    = (rawArtist || mainArtist).replace(/\p{S}/gu, '').trim();
  const searchArtist = ytmArtist;

  const escRe = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

  const searchTitle = cleanTitle;

  const variantFilter = buildVariantFilter(searchTitle);
  const titleKws      = buildTitleKeywords(searchTitle);
  const variantPart   = variantFilter ? ` & ${variantFilter}` : '';
  const titleKwPart   = titleKws.length ? titleKws.map(k => ` & title ~= (?i)${k}`).join('') : '';

  // Soft channel check: first word of original artist name used in early cascade steps to avoid
  // downloading a same-title song by a different artist (e.g. wrong "Moonlight" for N!GHT).
  // Skip when artistFirstWord is non-ASCII (Japanese kanji/kana): YouTube channel names are
  // almost never in kanji, so the filter would reject every valid result.
  const artistFirstWord = (rawArtist || mainArtist).split(/[\s,/]+/)[0];
  const artistChannelPart = (artistFirstWord.length >= 3 && !JP_RE.test(artistFirstWord))
    ? ` & channel ~= (?i)${escRe(artistFirstWord)}` : '';

  // Use a space (not " - ") between artist and title: this matches how users type
  // in the YTM search box and ranks the exact official track first. A dash skews
  // YTM ranking toward a same-named artist's full discography.
  const query = `${ytmArtist} ${searchTitle}`;

  const baseArgs = [
    '--extract-audio',
    '--audio-format',   CONFIG.audioFmt,
    '--audio-quality',  CONFIG.audioQuality,
    '-o',               outputTemplate,
    '--no-playlist',
    '--no-warnings',
    '--newline',
    // Hard upper size limit — prevents saving a wrong 28-min video as a ~52 MB mp3
    '--max-filesize',   '30M',
    // Use the web_music player client (falling back to default) so YouTube Music
    // "Art Track" / topic uploads are reachable. The android_vr client returns
    // "This video is not available" for many YTM tracks that play fine in the browser.
    '--extractor-args', 'youtube:player_client=web_music,default',
  ];

  const spawnYtdlp = (args) => new Promise((resolve, reject) => {
    const proc = spawn(CONFIG.ytdlpBin, args, { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
    let stderr = '', lastPct = '';
    proc.stdout.on('data', d => {
      d.toString().split('\n').filter(l => l.trim()).forEach(line => {
        if (line.includes('%') && line === lastPct) return;
        if (line.includes('%')) lastPct = line;
        log(jobId, line.trim());
      });
    });
    proc.stderr.on('data', d => {
      stderr += d.toString();
      d.toString().split('\n')
        .filter(l => l.trim() && !l.startsWith('[debug]'))
        .forEach(l => log(jobId, l.trim()));
    });
    proc.on('error', resolve);
    proc.on('close', code => {
      // Treat all non-zero exits as "no result" so the cascade continues.
      // Fatal errors (network, bad args) are logged via stderr above; hasAudio() decides outcome.
      if (code !== 0 && code !== 101) log(jobId, `⚠️  yt-dlp exit ${code}: ${stderr.slice(-200).trim()}`);
      resolve();
    });
  });

  // Use original (raw) artist for YTM URL — better ranking since channel is in original language.
  // Append the "Songs" filter param so results are restricted to YTM's Songs shelf
  // (no videos/playlists/clips) — the documented way to song-search YTM. This is cleaner
  // for normal tracks; for ambiguous artist names yt-dlp's search may still not surface the
  // exact official track (a known extractor limitation — use the direct link in that case).
  const YTM_SONGS_PARAM = 'EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D';
  const ytmQuery = `${rawArtist || mainArtist} ${searchTitle}`;
  const ytmUrl = `https://music.youtube.com/search?q=${encodeURIComponent(ytmQuery)}&params=${YTM_SONGS_PARAM}`;
  const expectedAudio = path.join(dir, `${fileBase}.${CONFIG.audioFmt}`);
  const hasAudio = () => fs.existsSync(expectedAudio);

  // Direct URL download (e.g. SoundCloud) — skip the ytsearch cascade entirely
  if (sourceUrl) {
    log(jobId, `🔗 Direkter Download: ${sourceUrl}`);
    await spawnYtdlp([sourceUrl, '--no-playlist', ...baseArgs]);
    const audioFile = hasAudio() ? expectedAudio : null;
    if (!audioFile) { log(jobId, '⚠️  Download fehlgeschlagen'); return { success: true, dir }; }
    return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title: searchTitle });
  }

  // Fetch YTM search result URLs as a flat list (no channel expansion) and return only
  // actual video URLs (watch?v=…). This prevents yt-dlp from following an artist channel
  // result and treating its uploads as the search result set.
  //
  // When YTM returns an artist channel as the top hit, yt-dlp prints the channel's
  // browse URL (/browse/UC…) instead of video URLs. We detect this on stdout and
  // return [] so the caller falls through to the ytsearch fallback.
  // NOTE: do NOT check for [youtube:tab] in stderr — that fires for all YTM search
  // page processing, not just channel expansion, and would wrongly return [] always.
  const getYtmVideoUrls = (maxItems) => new Promise(resolve => {
    const urls = [];
    let channelExpanded = false;
    const proc = spawn(CONFIG.ytdlpBin, [
      ytmUrl,
      '--flat-playlist',
      '--playlist-items', `1-${maxItems}`,
      '--print', 'webpage_url',
      '--no-warnings',
      // Without the web_music client, yt-dlp parses the YTM search page as a
      // generic YouTube search and returns unrelated videos instead of the
      // native Songs shelf (where the actual artist track lives).
      '--extractor-args', 'youtube:player_client=web_music,default',
    ], { env: { ...process.env, PYTHONUNBUFFERED: '1' } });
    proc.stdout.on('data', d => {
      const text = d.toString();
      if (text.includes('/browse/UC')) { channelExpanded = true; return; }
      if (!channelExpanded) {
        text.split('\n').forEach(line => {
          const u = line.trim();
          if (u.includes('watch?v=')) urls.push(u);
        });
      }
    });
    proc.on('error', () => resolve([]));
    proc.on('close', () => {
      if (channelExpanded) log(jobId, `⚠️  YTM: Artist-Channel in Suchergebnissen — überspringe`);
      else log(jobId, `📡 YTM: ${urls.length} Ergebnis(se) gefunden`);
      resolve(channelExpanded ? [] : urls);
    });
  });

  const mf = (dur, variant, strict = false) => {
    let durPart;
    if (dur) {
      durPart = `duration > ${dur.min} & duration < ${dur.max} & `;
    } else if (durationMs) {
      // Even when relaxing the window (Mittelweg), keep a 2× cap so a 28-min video can't slip through
      durPart = `duration < ${Math.round(durationMs / 1000 * 2)} & `;
    } else {
      durPart = '';
    }
    return ['--match-filter',
      `${durPart}!is_live${titleKwPart}${strict ? artistChannelPart : ''}${variant ? variantPart : ''}`];
  };

  const PI_FILTERED    = ['--playlist-items', '1-5',  '--max-downloads', '1'];
  const PI_FILTERED_10 = ['--playlist-items', '1-10', '--max-downloads', '1'];
  const PI_UNFILTERED  = ['--playlist-items', '1'];

  // Fetch YTM URLs once upfront so every cascade step filters the SAME result set.
  // Re-fetching produced different orderings, causing the correct video (found in step 1
  // but failing the channel filter) to disappear by step 3 and a wrong one to slip through.
  const ytmUrls = await getYtmVideoUrls(10);
  const tryYtm = async (n, mfArgs) => {
    const urls = n ? ytmUrls.slice(0, n) : ytmUrls;
    if (!urls.length) return;
    await spawnYtdlp([...urls, '--max-downloads', '1', ...mfArgs, ...baseArgs]);
  };

  if (durationMs && durationMs > 30000) {
    const durationSec = Math.round(durationMs / 1000);
    const margin = Math.max(10, Math.round(durationSec * 0.10));
    const dur = { min: Math.max(30, durationSec - margin), max: durationSec + margin };

    log(jobId, `🔍 Suche YT Music (~${durationSec}s ±${margin}s): "${query}"`);
    // 1. YTM top-5 — strict: duration + title + variant + channel check
    await tryYtm(5, mf(dur, true, true));
    if (!hasAudio()) {
      // 2. YT top-5 — strict: same filters including channel check
      log(jobId, '⚠️  Kein YTM-Treffer — Fallback zu YouTube…');
      await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(dur, true, true), ...baseArgs]);
    }
    if (!hasAudio()) {
      // 3. YTM all-10 — no channel check (label channels, etc.)
      log(jobId, '⚠️  Erweitere Suche auf top 10 mit Dauerbeschränkung…');
      await tryYtm(0, mf(dur, true));
    }
    if (!hasAudio()) {
      // 4. YT top-10 — no channel check
      await spawnYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(dur, true), ...baseArgs]);
    }
    // No duration match. "Mittelweg": relax the duration constraint but REQUIRE the artist
    // channel (+ title). This catches legit tracks whose YT upload duration drifts from the
    // iTunes value, while never grabbing a same-titled video by someone else. If nothing
    // matches title+channel we download NOTHING and the job fails cleanly (see end of fn) —
    // far better than silently saving the wrong song (e.g. a "recognize" fan clip).
    if (!hasAudio()) {
      // 5. YTM all — no duration, but channel + title required
      log(jobId, '⚠️  Kein Treffer im Dauerbereich — suche ohne Dauer, aber mit Künstler-Channel…');
      await tryYtm(0, mf(null, true, true));
    }
    if (!hasAudio()) {
      // 6. YT top-10 — no duration, but channel + title required
      await spawnYtdlp([`ytsearch10:${query}`, ...PI_FILTERED_10, ...mf(null, true, true), ...baseArgs]);
    }
  } else {
    log(jobId, `🔍 Suche YT Music: "${query}"`);
    // 1. YTM top-5 — strict: title + variant + channel check
    await tryYtm(5, mf(null, true, true));
    if (!hasAudio()) {
      // 2. YT top-5 — strict
      await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true, true), ...baseArgs]);
    }
    if (!hasAudio()) {
      // 3. YTM all-10 — no channel check (catches label/distributor channels)
      await tryYtm(0, mf(null, true));
    }
    if (!hasAudio()) {
      // 4. YT top-5 — no channel check
      await spawnYtdlp([`ytsearch5:${query}`, ...PI_FILTERED, ...mf(null, true), ...baseArgs]);
    }
    // Mittelweg (no-duration): after step 4, do NOT fall through to title-only or
    // no-filter grabs — those pull wrong covers (observed: Yorushika → Piano Echoes).
    // Fail cleanly, same as the with-duration branch. Use the Direktlink instead.
  }

  const audioFile = hasAudio() ? expectedAudio : null;
  if (!audioFile) {
    log(jobId, '⚠️  Nicht eindeutig gefunden — bitte Direktlink im Importer nutzen');
    return { success: false, notFound: true, dir };
  }

  return embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title: searchTitle });
}

// Shared ffmpeg metadata embedding — used by both yt-dlp and squid paths
async function embedMetadata({ jobId, audioFile, dir, trackPrefix, safeTitle, rawArtist, artist, mainArtist, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, title }) {
  const coverPath = await downloadArtwork(artworkUrl, dir);
  const hasCover = !!(coverPath && fs.existsSync(coverPath));

  log(jobId, `🏷  Metadaten + Cover → ${path.basename(audioFile)}`);

  const fileExt = path.extname(audioFile);
  const tmpFile = audioFile.slice(0, -fileExt.length) + '.metatmp' + fileExt;
  await new Promise(resolve => {
    const ffArgs = ['-i', audioFile];
    if (hasCover) ffArgs.push('-i', coverPath);
    ffArgs.push('-map', '0:a');
    if (hasCover) ffArgs.push('-map', '1:v');
    // Main tags: store original text (Japanese or whatever was passed in).
    // rawArtist = original Spotify/source artist (may be Japanese); artist = iTunes-resolved (may be English).
    // Use rawArtist for the per-track artist tag; mainArtist for album_artist so all tracks
    // in an album get the same value (prevents Navidrome album-splitting).
    const tagArtist      = navidromeArtist(rawArtist || artist);
    const tagAlbumArtist = mainArtist.replace(/\s+&\s+/g, ', ');
    // Sort tags: romaji for display / sorting in apps that support them (TITLESORT, ARTISTSORT, etc.)
    // artist is already the iTunes-resolved English name when available — use directly (more accurate
    // than kuromoji). For non-iTunes tracks romajiForStorage acts as kuromoji fallback.
    const sortTitle       = romajiForStorage(title)                                        || title;
    const sortArtist      = romajiForStorage(navidromeArtist(artist))                     || navidromeArtist(artist);
    const sortAlbumArtist = romajiForStorage(mainArtist.replace(/\s+&\s+/g, ', '))        || mainArtist.replace(/\s+&\s+/g, ', ');
    const sortAlbum       = album ? (romajiForStorage(album) || album) : null;
    ffArgs.push(
      '-c',             'copy',
      '-map_metadata',  '-1',
      '-id3v2_version', '3',
      '-metadata', `title=${title}`,
      '-metadata', `artist=${tagArtist}`,
      '-metadata', `album_artist=${tagAlbumArtist}`,
    );
    if (album)      ffArgs.push('-metadata', `album=${album}`);
    // Sort tags — only write when they differ from main (no-op for non-Japanese tracks)
    if (sortTitle       !== title)         ffArgs.push('-metadata', `TITLESORT=${sortTitle}`);
    if (sortArtist      !== tagArtist)     ffArgs.push('-metadata', `ARTISTSORT=${sortArtist}`);
    if (sortAlbumArtist !== tagAlbumArtist) ffArgs.push('-metadata', `ALBUMARTISTSORT=${sortAlbumArtist}`);
    if (sortAlbum && sortAlbum !== album)  ffArgs.push('-metadata', `ALBUMSORT=${sortAlbum}`);
    if (year)        ffArgs.push('-metadata', `date=${String(year)}`);
    if (trackNum)    ffArgs.push('-metadata', `track=${totalTracks ? `${trackNum}/${totalTracks}` : String(trackNum)}`);
    if (totalTracks) ffArgs.push('-metadata', `totaltracks=${String(totalTracks)}`);
    if (genre)       ffArgs.push('-metadata', `genre=${genre}`);
    if (discNum)     ffArgs.push('-metadata', `disc=${discTotal ? `${discNum}/${discTotal}` : String(discNum)}`);
    if (compilation) ffArgs.push('-metadata', `compilation=1`);
    if (explicit)    ffArgs.push('-metadata', 'ITUNESADVISORY=1');
    if (hasCover) {
      ffArgs.push(
        '-metadata:s:v', 'title=Album cover',
        '-metadata:s:v', 'comment=Cover (front)',
      );
    }
    ffArgs.push(tmpFile, '-y');

    const ff = spawn('ffmpeg', ffArgs);
    let ffErr = '';
    ff.stderr.on('data', d => ffErr += d.toString());
    ff.on('error', () => resolve());
    ff.on('close', code => {
      if (code === 0 && fs.existsSync(tmpFile)) {
        fs.renameSync(tmpFile, audioFile);
        if (hasCover) try { fs.unlinkSync(coverPath); } catch {}
        const base = path.basename(audioFile, path.extname(audioFile));
        for (const ext of ['webm', 'webm.part', 'm4a', 'opus']) {
          try { fs.unlinkSync(path.join(dir, `${base}.${ext}`)); } catch {}
        }
        log(jobId, `✅ Fertig${hasCover ? ' (Cover 1000x1000 eingebettet)' : ''}`);
      } else {
        try { fs.unlinkSync(tmpFile); } catch {}
        log(jobId, `⚠️  ffmpeg code ${code}`);
        if (ffErr) log(jobId, ffErr.slice(-200));
      }
      resolve();
    });
  });

  return { success: true, dir };
}

module.exports = { downloadTrack, embedMetadata };