feat: romaji index for Japanese kanji song search
Backend: - Add /api/romaji-index endpoint using kuromoji morphological analyzer - Fetches all Navidrome songs, converts kanji → hiragana readings → romaji - Only indexes songs with Japanese characters; caches for 20 min - /api/romaji-index DELETE clears cache (force rebuild) Frontend: - fetchRomajiIndex() in api.js: lazy-fetches and caches the index - doLocalSearch: when query is romaji (ASCII), queries romaji index in parallel with Navidrome search3 and merges+deduplicates results Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
369a5e82b3
commit
aa239f8e6a
4 files changed
+176
-6
No files matched your search
+138
-4
@@ -8,17 +8,151 @@
|
|||||||
* - ffmpeg installed
|
* - ffmpeg installed
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const express = require('express');
|
const express = require('express');
|
||||||
const { execSync } = require('child_process');
|
const { execSync } = require('child_process');
|
||||||
const path = require('path');
|
const { createHash } = require('crypto');
|
||||||
const fs = require('fs');
|
const path = require('path');
|
||||||
const https = require('https');
|
const fs = require('fs');
|
||||||
|
const https = require('https');
|
||||||
|
|
||||||
const CONFIG = require('./lib/config');
|
const CONFIG = require('./lib/config');
|
||||||
const { jobs, updateJob, log } = require('./lib/jobs');
|
const { jobs, updateJob, log } = require('./lib/jobs');
|
||||||
const { lookupItunesTrackMeta, fetchItunesTracks, searchItunesAlbumTracks } = require('./lib/metadata');
|
const { lookupItunesTrackMeta, fetchItunesTracks, searchItunesAlbumTracks } = require('./lib/metadata');
|
||||||
const { downloadTrack, downloadAlbumTracks } = require('./lib/downloader');
|
const { downloadTrack, downloadAlbumTracks } = require('./lib/downloader');
|
||||||
|
|
||||||
|
// ─── ROMAJI INDEX ─────────────────────────────────────────────────────────────
|
||||||
|
// Converts kanji/kana → romaji so users can search Japanese songs phonetically.
|
||||||
|
|
||||||
|
let _tokenizer = null;
|
||||||
|
try {
|
||||||
|
const kuromoji = require('kuromoji');
|
||||||
|
kuromoji.builder({ dicPath: path.join(require.resolve('kuromoji'), '../../dict') })
|
||||||
|
.build((err, t) => {
|
||||||
|
if (err) console.warn('⚠️ kuromoji failed to load:', err.message);
|
||||||
|
else { _tokenizer = t; console.log('✅ kuromoji tokenizer ready'); }
|
||||||
|
});
|
||||||
|
} catch { console.warn('⚠️ kuromoji not installed — run: cd backend && npm install'); }
|
||||||
|
|
||||||
|
// katakana→hiragana shift (kuromoji readings come back in katakana)
|
||||||
|
function _kataToHira(s) { return s.replace(/[ァ-ヶ]/g, c => String.fromCharCode(c.charCodeAt(0) - 0x60)); }
|
||||||
|
|
||||||
|
// Full hiragana → romaji table (includes digraphs + っ doubling)
|
||||||
|
const _H2R = {
|
||||||
|
'あ':'a','い':'i','う':'u','え':'e','お':'o',
|
||||||
|
'か':'ka','き':'ki','く':'ku','け':'ke','こ':'ko',
|
||||||
|
'さ':'sa','し':'shi','す':'su','せ':'se','そ':'so',
|
||||||
|
'た':'ta','ち':'chi','つ':'tsu','て':'te','と':'to',
|
||||||
|
'な':'na','に':'ni','ぬ':'nu','ね':'ne','の':'no',
|
||||||
|
'は':'ha','ひ':'hi','ふ':'fu','へ':'he','ほ':'ho',
|
||||||
|
'ま':'ma','み':'mi','む':'mu','め':'me','も':'mo',
|
||||||
|
'や':'ya','ゆ':'yu','よ':'yo',
|
||||||
|
'ら':'ra','り':'ri','る':'ru','れ':'re','ろ':'ro',
|
||||||
|
'わ':'wa','を':'wo','ん':'n',
|
||||||
|
'が':'ga','ぎ':'gi','ぐ':'gu','げ':'ge','ご':'go',
|
||||||
|
'ざ':'za','じ':'ji','ず':'zu','ぜ':'ze','ぞ':'zo',
|
||||||
|
'だ':'da','ぢ':'di','づ':'du','で':'de','ど':'do',
|
||||||
|
'ば':'ba','び':'bi','ぶ':'bu','べ':'be','ぼ':'bo',
|
||||||
|
'ぱ':'pa','ぴ':'pi','ぷ':'pu','ぺ':'pe','ぽ':'po',
|
||||||
|
'きゃ':'kya','きゅ':'kyu','きょ':'kyo',
|
||||||
|
'しゃ':'sha','しゅ':'shu','しょ':'sho',
|
||||||
|
'ちゃ':'cha','ちゅ':'chu','ちょ':'cho',
|
||||||
|
'にゃ':'nya','にゅ':'nyu','にょ':'nyo',
|
||||||
|
'ひゃ':'hya','ひゅ':'hyu','ひょ':'hyo',
|
||||||
|
'みゃ':'mya','みゅ':'myu','みょ':'myo',
|
||||||
|
'りゃ':'rya','りゅ':'ryu','りょ':'ryo',
|
||||||
|
'ぎゃ':'gya','ぎゅ':'gyu','ぎょ':'gyo',
|
||||||
|
'じゃ':'ja','じゅ':'ju','じょ':'jo',
|
||||||
|
'びゃ':'bya','びゅ':'byu','びょ':'byo',
|
||||||
|
'ぴゃ':'pya','ぴゅ':'pyu','ぴょ':'pyo',
|
||||||
|
'ふぁ':'fa','ふぃ':'fi','ふぇ':'fe','ふぉ':'fo',
|
||||||
|
'てぃ':'ti','でぃ':'di','とぅ':'tu','どぅ':'du',
|
||||||
|
'ゐ':'i','ゑ':'e','ヴ':'vu',
|
||||||
|
};
|
||||||
|
function _hiraToRomaji(str) {
|
||||||
|
let out = '', i = 0;
|
||||||
|
while (i < str.length) {
|
||||||
|
if (str[i] === 'っ') {
|
||||||
|
const nxt = _H2R[str.slice(i+1,i+3)] || _H2R[str[i+1]] || '';
|
||||||
|
out += nxt[0] || ''; i++; continue;
|
||||||
|
}
|
||||||
|
const two = _H2R[str.slice(i,i+2)];
|
||||||
|
if (two) { out += two; i += 2; continue; }
|
||||||
|
out += _H2R[str[i]] ?? str[i]; i++;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
const JP_RE = /[-ヿ㐀-䶿一-鿿-]/;
|
||||||
|
function _toRomaji(text) {
|
||||||
|
if (!text || !_tokenizer || !JP_RE.test(text)) return text || '';
|
||||||
|
const tokens = _tokenizer.tokenize(text);
|
||||||
|
const hira = tokens.map(t => {
|
||||||
|
const r = t.reading;
|
||||||
|
return (r && r !== '*') ? _kataToHira(r) : t.surface_form;
|
||||||
|
}).join('');
|
||||||
|
return _hiraToRomaji(hira).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cache: key = "ndUrl|user" → { ts, index }
|
||||||
|
const _romajiCache = new Map();
|
||||||
|
const ROMAJI_TTL = 20 * 60 * 1000; // 20 min
|
||||||
|
|
||||||
|
async function _fetchNdSongs(ndUrl, user, pass) {
|
||||||
|
const salt = Math.random().toString(36).slice(2, 10);
|
||||||
|
const token = createHash('md5').update(pass + salt).digest('hex');
|
||||||
|
const base = `${ndUrl}/rest/search3.view?u=${encodeURIComponent(user)}&t=${token}&s=${salt}&v=1.16.1&c=novasonic&f=json&albumCount=0&artistCount=0`;
|
||||||
|
const songs = [];
|
||||||
|
let offset = 0;
|
||||||
|
while (true) {
|
||||||
|
const r = await fetch(`${base}&query=&songCount=500&songOffset=${offset}`).then(r => r.json());
|
||||||
|
const page = r['subsonic-response']?.searchResult3?.song || [];
|
||||||
|
songs.push(...page);
|
||||||
|
if (page.length < 500) break;
|
||||||
|
offset += 500;
|
||||||
|
}
|
||||||
|
return songs;
|
||||||
|
}
|
||||||
|
|
||||||
|
// GET /api/romaji-index?ndUrl=...&user=...&pass=...
|
||||||
|
app.get('/api/romaji-index', async (req, res) => {
|
||||||
|
const { ndUrl, user, pass } = req.query;
|
||||||
|
if (!ndUrl || !user || !pass) return res.status(400).json({ error: 'missing params' });
|
||||||
|
if (!_tokenizer) return res.status(503).json({ error: 'tokenizer not ready — try again in a few seconds' });
|
||||||
|
|
||||||
|
const key = `${ndUrl}|${user}`;
|
||||||
|
const cached = _romajiCache.get(key);
|
||||||
|
if (cached && Date.now() - cached.ts < ROMAJI_TTL) return res.json(cached.index);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const songs = await _fetchNdSongs(ndUrl, user, pass);
|
||||||
|
const index = songs
|
||||||
|
.filter(s => JP_RE.test(s.title) || JP_RE.test(s.artist) || JP_RE.test(s.album))
|
||||||
|
.map(s => ({
|
||||||
|
id: s.id,
|
||||||
|
r: [_toRomaji(s.title), _toRomaji(s.artist), _toRomaji(s.album)].join(' '),
|
||||||
|
title: s.title,
|
||||||
|
artist: s.artist || '',
|
||||||
|
album: s.album || '',
|
||||||
|
coverArt: s.coverArt || '',
|
||||||
|
duration: s.duration || 0,
|
||||||
|
albumId: s.albumId || '',
|
||||||
|
artistId: s.artistId || '',
|
||||||
|
}));
|
||||||
|
_romajiCache.set(key, { ts: Date.now(), index });
|
||||||
|
console.log(`✅ Romaji index built: ${index.length} JP songs (${songs.length} total)`);
|
||||||
|
res.json(index);
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Romaji index error:', e);
|
||||||
|
res.status(500).json({ error: e.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// DELETE /api/romaji-index — force rebuild on next request
|
||||||
|
app.delete('/api/romaji-index', (req, res) => {
|
||||||
|
_romajiCache.clear();
|
||||||
|
res.json({ ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
// ─── INIT ─────────────────────────────────────────────────────────────────────
|
// ─── INIT ─────────────────────────────────────────────────────────────────────
|
||||||
const app = express();
|
const app = express();
|
||||||
app.use((req, res, next) => {
|
app.use((req, res, next) => {
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"name": "novasonic-backend",
|
||||||
|
"private": true,
|
||||||
|
"dependencies": {
|
||||||
|
"kuromoji": "^0.1.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
+19
@@ -39,6 +39,25 @@ export async function ndAuth() {
|
|||||||
return S._ndToken;
|
return S._ndToken;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── ROMAJI INDEX ───
|
||||||
|
// Fetched once per session from the backend, cached in memory.
|
||||||
|
let _romajiIndex = null;
|
||||||
|
let _romajiIndexFetch = null; // in-flight promise dedup
|
||||||
|
|
||||||
|
export async function fetchRomajiIndex() {
|
||||||
|
if (!S.cfg.backendUrl || !S.cfg.url || !S.cfg.user || !S.cfg.pass) return null;
|
||||||
|
if (_romajiIndex) return _romajiIndex;
|
||||||
|
if (_romajiIndexFetch) return _romajiIndexFetch;
|
||||||
|
_romajiIndexFetch = fetch(
|
||||||
|
`${S.cfg.backendUrl}/api/romaji-index?ndUrl=${encodeURIComponent(S.cfg.url)}&user=${encodeURIComponent(S.cfg.user)}&pass=${encodeURIComponent(S.cfg.pass)}`
|
||||||
|
).then(r => r.ok ? r.json() : null)
|
||||||
|
.then(idx => { _romajiIndex = idx; _romajiIndexFetch = null; return idx; })
|
||||||
|
.catch(() => { _romajiIndexFetch = null; return null; });
|
||||||
|
return _romajiIndexFetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function invalidateRomajiIndex() { _romajiIndex = null; }
|
||||||
|
|
||||||
// ─── ITUNES ───
|
// ─── ITUNES ───
|
||||||
// Route through backend proxy to avoid CORS; fall back to direct if no backend configured.
|
// Route through backend proxy to avoid CORS; fall back to direct if no backend configured.
|
||||||
function _itunesBase() {
|
function _itunesBase() {
|
||||||
|
|||||||
+12
-2
@@ -6,6 +6,7 @@ import { playQueue, dlSongLocal, playPreview } from './player.js';
|
|||||||
import { showCtxMenu, openEditPlModal, openDlModal, closeSidebar, switchNavMode } from './ui.js';
|
import { showCtxMenu, openEditPlModal, openDlModal, closeSidebar, switchNavMode } from './ui.js';
|
||||||
import { closeNowPlaying, closeNpMenu } from './nowplaying.js';
|
import { closeNowPlaying, closeNpMenu } from './nowplaying.js';
|
||||||
import { makeCard, makeAlbumCard, makeSongItem, makeGlobalTrackItem, mkSection } from './components.js';
|
import { makeCard, makeAlbumCard, makeSongItem, makeGlobalTrackItem, mkSection } from './components.js';
|
||||||
|
import { fetchRomajiIndex } from './api.js';
|
||||||
import { loadSpotifyImporter } from './spotify.js';
|
import { loadSpotifyImporter } from './spotify.js';
|
||||||
|
|
||||||
// ─── NAVIGATION HISTORY ───
|
// ─── NAVIGATION HISTORY ───
|
||||||
@@ -723,16 +724,25 @@ export async function doLocalSearch(q) {
|
|||||||
el.innerHTML = '<div class="spinner"></div>';
|
el.innerHTML = '<div class="spinner"></div>';
|
||||||
try {
|
try {
|
||||||
const hiragana = await _romajiToHiragana(q);
|
const hiragana = await _romajiToHiragana(q);
|
||||||
const [r, r2] = await Promise.all([
|
const isRomaji = /^[a-z\s'-]+$/i.test(q.trim());
|
||||||
|
const [r, r2, romajiIdx] = await Promise.all([
|
||||||
api('search3', `&query=${encodeURIComponent(q)}&songCount=30&albumCount=10&artistCount=6`),
|
api('search3', `&query=${encodeURIComponent(q)}&songCount=30&albumCount=10&artistCount=6`),
|
||||||
hiragana ? api('search3', `&query=${encodeURIComponent(hiragana)}&songCount=20&albumCount=6&artistCount=4`) : Promise.resolve(null),
|
hiragana ? api('search3', `&query=${encodeURIComponent(hiragana)}&songCount=20&albumCount=6&artistCount=4`) : Promise.resolve(null),
|
||||||
|
isRomaji ? fetchRomajiIndex() : Promise.resolve(null),
|
||||||
]);
|
]);
|
||||||
const res = r.searchResult3 || {};
|
const res = r.searchResult3 || {};
|
||||||
const res2 = r2?.searchResult3 || {};
|
const res2 = r2?.searchResult3 || {};
|
||||||
el.innerHTML = '';
|
el.innerHTML = '';
|
||||||
const artists = _mergeUniq(res.artist||[], res2.artist);
|
const artists = _mergeUniq(res.artist||[], res2.artist);
|
||||||
const albums = _mergeUniq(res.album ||[], res2.album);
|
const albums = _mergeUniq(res.album ||[], res2.album);
|
||||||
const songs = _mergeUniq(res.song ||[], res2.song);
|
|
||||||
|
// Merge Navidrome results with romaji index matches
|
||||||
|
const qLow = q.trim().toLowerCase();
|
||||||
|
const romajiHits = romajiIdx
|
||||||
|
? romajiIdx.filter(e => e.r.includes(qLow))
|
||||||
|
: [];
|
||||||
|
const songs = _mergeUniq(_mergeUniq(res.song||[], res2.song), romajiHits);
|
||||||
|
|
||||||
if(!artists.length && !albums.length && !songs.length) {
|
if(!artists.length && !albums.length && !songs.length) {
|
||||||
el.innerHTML = '<div class="empty-state"><div class="icon">🔍</div><p>Keine Ergebnisse</p></div>'; return;
|
el.innerHTML = '<div class="empty-state"><div class="icon">🔍</div><p>Keine Ergebnisse</p></div>'; return;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user