Files
NovaSonic/backend/lib/squid.js
T
HoneyandClaude Sonnet 4.6 63ff8920bf fix: repair Japanese song download pipeline
Five separate root causes fixed:

1. buildTitleKeywords (ytdlp.js): For Japanese titles \w strips all
   kana/kanji leaving no keywords. Now uses the first whitespace-
   delimited segment as a literal regex keyword so --match-filter
   title ~= requires the YTM video to contain the Japanese text.

2. Tokenizer wait (ytdlp.js): If kuromoji hasn't finished loading when
   a Japanese download starts, romajiForStorage returns the original
   Japanese text unchanged — producing Japanese folder/filenames and
   skipping sort tags. Now waits up to 30 s for the tokenizer.

3. titleOnlyMatch guard (metadata.js): iTunes titleOnlyMatch (no artist
   check) was used as a fallback for all searches. For Japanese
   titles/artists this matched wrong US songs with identical titles
   (common in anime OSTs). Now titleOnlyMatch is skipped when the
   search title or artist contains Japanese characters.

4. squid.js rawArtist: rawArtist was not in squid's signature and
   never forwarded to embedMetadata, so Qobuz-downloaded J-Pop tracks
   stored the iTunes-resolved English name as artist instead of the
   Japanese original. Fixed by accepting and forwarding rawArtist.

5. squid.js romaji paths: Folder and filename used raw artist/title
   with no romaji conversion, creating Japanese-named paths
   inconsistent with the yt-dlp path. Now uses romajiForStorage same
   as ytdlp.js.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-29 01:40:46 +02:00

293 lines
11 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
const crypto = require('crypto');
const https = require('https');
const http = require('http');
const path = require('path');
const fs = require('fs');
const { spawn } = require('child_process');
const CONFIG = require('./config');
const { log } = require('./jobs');
const {
sanitize, navidromeArtist,
downloadArtwork, fetchLyrics, findExistingTrack,
} = require('./metadata');
const { embedMetadata } = require('./ytdlp');
const { romajiForStorage } = require('./romaji');
const SQUID_BASE = 'https://qobuz.squid.wtf';
// Qobuz quality codes: mp3=5 (320kbps), flac=6 (16/44), hi-res=7 (24/96)
const QUALITY_CODE = { mp3: '5', flac: '6', 'hi-res': '7' };
function squidExt() {
return CONFIG.squidQuality === 'mp3' ? 'mp3' : 'flac';
}
// ─── ALTCHA cookie cache ─────────────────────────────────────────────────────
let _captchaCookie = null;
let _captchaAt = 0;
const CAPTCHA_TTL = 27 * 60 * 1000; // refresh 1 min before server's 28-min expiry
// Standard ALTCHA PoW solver: find number such that SHA-256(salt + number) == challenge
async function solveAltcha(challenge) {
const { algorithm = 'SHA-256', challenge: target, salt, maxnumber = 1_000_000 } = challenge;
if (algorithm !== 'SHA-256') throw new Error(`ALTCHA: unsupported algorithm ${algorithm}`);
const startMs = Date.now();
for (let number = 0; number <= maxnumber; number++) {
const hash = crypto.createHash('sha256').update(salt + String(number)).digest('hex');
if (hash === target) {
return { number, time: Date.now() - startMs };
}
if (number % 10_000 === 0) await new Promise(r => setImmediate(r));
}
throw new Error('ALTCHA: no solution found within maxnumber');
}
function httpsGet(url) {
return new Promise((resolve, reject) => {
const mod = url.startsWith('https') ? https : http;
mod.get(url, { headers: { 'User-Agent': 'NovaSonic/1.0' } }, res => {
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: d }));
}).on('error', reject);
});
}
function httpsPost(url, body, extraHeaders) {
return new Promise((resolve, reject) => {
const payload = JSON.stringify(body);
const parsed = new URL(url);
const opts = {
hostname: parsed.hostname,
port: parsed.port || 443,
path: parsed.pathname + parsed.search,
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(payload),
'User-Agent': 'NovaSonic/1.0',
...extraHeaders,
},
};
const req = https.request(opts, res => {
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: d }));
});
req.on('error', reject);
req.write(payload);
req.end();
});
}
async function getAltchaCookie() {
if (_captchaCookie && Date.now() - _captchaAt < CAPTCHA_TTL) return _captchaCookie;
// 1. Fetch challenge — standard ALTCHA returns { algorithm, challenge, salt, signature, maxnumber }
const chRes = await httpsGet(`${SQUID_BASE}/api/altcha/challenge`);
if (chRes.status !== 200) throw new Error(`ALTCHA challenge HTTP ${chRes.status}: ${chRes.body.slice(0,200)}`);
const chBody = JSON.parse(chRes.body);
const challenge = chBody.parameters ?? chBody; // unwrap { parameters: {...} } if present
log('altcha', `🔐 ALTCHA challenge: alg=${challenge.algorithm} max=${challenge.maxnumber}`);
// 2. Solve PoW
const solution = await solveAltcha(challenge);
log('altcha', `🔐 ALTCHA gelöst in ${solution.time}ms (number=${solution.number})`);
// 3. Verify — standard ALTCHA payload: base64(JSON { algorithm, challenge, number, salt, signature })
const payloadObj = {
algorithm: challenge.algorithm || 'SHA-256',
challenge: challenge.challenge,
number: solution.number,
salt: challenge.salt,
signature: challenge.signature,
};
const payload = Buffer.from(JSON.stringify(payloadObj)).toString('base64');
const vRes = await httpsPost(`${SQUID_BASE}/api/altcha/verify`, { payload });
if (vRes.status !== 200) throw new Error(`ALTCHA verify HTTP ${vRes.status}: ${vRes.body.slice(0,200)}`);
// Parse Set-Cookie: captcha_verified_at=<value>; ...
const setCookie = [vRes.headers['set-cookie']].flat().filter(Boolean);
const cookiePair = setCookie.map(c => c.split(';')[0]).find(c => c.startsWith('captcha_verified_at='));
if (!cookiePair) throw new Error('ALTCHA verify: no cookie in response');
_captchaCookie = cookiePair;
_captchaAt = Date.now();
return _captchaCookie;
}
async function squidApiGet(endpoint) {
const cookie = await getAltchaCookie();
return new Promise((resolve, reject) => {
const url = `${SQUID_BASE}${endpoint}`;
const parsed = new URL(url);
const opts = {
hostname: parsed.hostname,
port: parsed.port || 443,
path: parsed.pathname + parsed.search,
method: 'GET',
headers: {
'Cookie': cookie,
'Token-Country': 'US',
'User-Agent': 'NovaSonic/1.0',
},
};
const req = https.request(opts, res => {
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.on('error', reject);
req.end();
});
}
// Search Qobuz via squid.wtf, return best matching track object or null
async function squidSearch(artist, title, durationMs) {
const query = encodeURIComponent(`${artist} ${title}`);
const res = await squidApiGet(`/api/get-music?q=${query}&offset=0`);
if (res.status !== 200) throw new Error(`squid search HTTP ${res.status}`);
const json = JSON.parse(res.body);
// Handle both {data:{tracks:{items:[...]}}} and {tracks:{items:[...]}} wrappers
const items = json?.data?.tracks?.items ?? json?.tracks?.items ?? [];
if (!items.length) return null;
const norm = s => (s || '').toLowerCase().trim();
const normT = norm(title);
const normA = norm(artist).split(/[,\s]/)[0];
// Score candidates: prefer exact title match + matching artist + duration proximity
const scored = items.map(t => {
const tTitle = norm(t.title);
const tArtist = norm(t.performer?.name || '');
const tDuration = (t.duration || 0) * 1000; // Qobuz uses seconds
let score = 0;
if (tTitle === normT) score += 100;
else if (tTitle.includes(normT)) score += 50;
if (tArtist.startsWith(normA)) score += 30;
if (durationMs && tDuration) {
const diff = Math.abs(tDuration - durationMs);
if (diff < 3000) score += 20; // within 3 s
else if (diff < 10000) score += 10; // within 10 s
else if (diff > 30000) score -= 20; // way off
}
return { track: t, score };
});
scored.sort((a, b) => b.score - a.score);
const best = scored[0];
// Reject if score is too low (no reasonable match)
if (best.score < 30) return null;
return best.track;
}
// Stream a URL to destPath (handles redirects via Location header)
async function downloadUrlToFile(url, destPath) {
return new Promise((resolve, reject) => {
const tryFetch = (u, hops) => {
if (hops > 5) { reject(new Error('Too many redirects')); return; }
const mod = u.startsWith('https') ? https : http;
const parsed = new URL(u);
const opts = {
hostname: parsed.hostname,
port: parsed.port || (u.startsWith('https') ? 443 : 80),
path: parsed.pathname + parsed.search,
method: 'GET',
headers: { 'User-Agent': 'NovaSonic/1.0' },
};
const req = mod.request(opts, res => {
if (res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 307) {
res.resume();
tryFetch(res.headers.location, hops + 1);
return;
}
if (res.statusCode !== 200) {
res.resume();
reject(new Error(`Download HTTP ${res.statusCode}`));
return;
}
const file = fs.createWriteStream(destPath);
res.pipe(file);
file.on('finish', () => file.close(resolve));
file.on('error', reject);
});
req.on('error', reject);
req.end();
};
tryFetch(url, 0);
});
}
// Download a single track via squid.wtf / Qobuz.
// Returns { success: true, dir, skipped? } on success, null if track not found on Qobuz.
async function downloadTrack({ jobId, artist, rawArtist, albumArtist, title, album, year, trackNum, totalTracks, discNum, discTotal, compilation, genre, artworkUrl, explicit, outputDir, durationMs }) {
const mainArtist = albumArtist || artist;
// Use romaji for folder/filename — same as ytdlp path so paths are consistent
const mainArtistRomaji = romajiForStorage(mainArtist) || mainArtist;
const titleRomaji = romajiForStorage(title) || title;
const albumRomaji = album ? (romajiForStorage(album) || album) : null;
const safeArtist = sanitize(mainArtistRomaji.split(/[,;&]/)[0].trim());
const safeAlbum = sanitize(albumRomaji || 'Singles');
const dir = path.join(outputDir || CONFIG.musicPath, safeArtist, safeAlbum);
fs.mkdirSync(dir, { recursive: true });
const safeTitle = sanitize(titleRomaji);
const trackPrefix = trackNum ? String(trackNum).padStart(2, '0') + ' - ' : '';
const ext = squidExt();
const existing = findExistingTrack(dir, trackPrefix, safeTitle);
if (existing) {
log(jobId, `⏭ Bereits vorhanden: ${path.basename(existing)} — übersprungen`);
return { success: true, dir, skipped: true };
}
log(jobId, `🦑 squid.wtf: suche "${artist} – ${title}"`);
// Solve ALTCHA if needed (cached for 27 min)
await getAltchaCookie();
// Find track on Qobuz
const searchArtist = mainArtist.replace(/[^\p{L}\p{N}\s,&\-]/gu, '').trim();
const track = await squidSearch(searchArtist, title, durationMs);
if (!track) {
log(jobId, `⚠️ squid.wtf: "${title}" nicht auf Qobuz gefunden`);
return null; // signals caller to fall back to yt-dlp
}
// Get download URL
const quality = QUALITY_CODE[CONFIG.squidQuality] || '5';
const dlRes = await squidApiGet(`/api/download-music?track_id=${track.id}&quality=${quality}`);
if (dlRes.status !== 200) throw new Error(`squid download-music HTTP ${dlRes.status}`);
const dlJson = JSON.parse(dlRes.body);
const streamUrl = dlJson?.data?.url ?? dlJson?.url;
if (!streamUrl) throw new Error('squid: no download URL in response');
// Download to a temp file, then embed metadata
const audioFile = path.join(dir, `${trackPrefix}${safeTitle}.${ext}`);
const tmpFile = audioFile + '.squidtmp';
log(jobId, `⬇️ Lade herunter (Qobuz ${CONFIG.squidQuality.toUpperCase()})…`);
try {
await downloadUrlToFile(streamUrl, tmpFile);
fs.renameSync(tmpFile, audioFile);
} catch (e) {
try { fs.unlinkSync(tmpFile); } catch {}
throw e;
}
// Use Qobuz artwork if no better URL was passed in
const finalArtwork = artworkUrl || track.album?.image?.large || track.album?.image?.small || '';
return embedMetadata({
jobId, audioFile, dir, trackPrefix, safeTitle,
rawArtist, artist, mainArtist, album, year, trackNum, totalTracks,
discNum, discTotal, compilation, genre,
artworkUrl: finalArtwork, explicit, title,
});
}
module.exports = { downloadTrack };