openchamber/packages/web/server/lib/tts/language-detect.js

/**
 * Language detection for text-to-speech voice selection.
 *
 * Picks the language a piece of chat text is written in so a TTS provider
 * can choose a matching voice or model. Deliberately small and dependency
 * free: the writing system decides most cases outright, and Latin-script
 * languages are told apart by function words and characteristic letters.
 * The answer is a best effort for voice selection, not a linguistic claim —
 * an unknown language falls back to English rather than failing.
 */

const SCRIPT_RANGES = [
  ['hangul', /[가-힯ᄀ-ᇿ㄰-㆏]/g],
  ['kana', /[぀-ヿ]/g],
  ['han', /[一-鿿㐀-䶿]/g],
  ['cyrillic', /[Ѐ-ӿ]/g],
  ['greek', /[Ͱ-Ͽ]/g],
  ['arabic', /[؀-ۿ]/g],
  ['hebrew', /[֐-׿]/g],
  ['thai', /[฀-๿]/g],
  ['devanagari', /[ऀ-ॿ]/g],
  ['latin', /[A-Za-zÀ-ɏ]/g],
];

const SCRIPT_LANGUAGE = {
  hangul: 'ko',
  greek: 'el',
  arabic: 'ar',
  hebrew: 'he',
  thai: 'th',
  devanagari: 'hi',
};

// Letters that only (or overwhelmingly) occur in one language of a script.
const LATIN_MARKERS = {
  pl: /[łęąńśźż]/i,
  cs: /[řěůťďň]/i,
  tr: /[ğışİ]/,
  pt: /[ãõ]/i,
  es: /[ñ¿¡]/,
  de: /[ß]/,
  fr: /[œ]/i,
  sv: /[å]/i,
};

// Frequent function words per language. Scored by whole-word hits; every
// list has the same length so scores stay comparable.
const STOPWORDS = {
  en: ['the', 'and', 'is', 'to', 'of', 'that', 'you', 'with', 'for', 'this', 'are', 'it', 'not', 'have', 'can', 'will', 'your', 'from', 'which', 'when'],
  de: ['und', 'der', 'die', 'das', 'ist', 'nicht', 'mit', 'ein', 'eine', 'auch', 'sich', 'auf', 'für', 'wird', 'werden', 'oder', 'aber', 'wenn', 'sind', 'kann'],
  fr: ['le', 'la', 'les', 'et', 'est', 'une', 'des', 'pour', 'que', 'qui', 'dans', 'pas', 'vous', 'sur', 'avec', 'sont', 'nous', 'cette', 'mais', 'plus'],
  es: ['el', 'la', 'los', 'las', 'que', 'es', 'una', 'por', 'para', 'con', 'del', 'como', 'pero', 'más', 'este', 'esta', 'son', 'tiene', 'puede', 'también'],
  it: ['il', 'la', 'che', 'di', 'è', 'una', 'per', 'non', 'con', 'del', 'della', 'come', 'sono', 'anche', 'questo', 'questa', 'gli', 'nel', 'più', 'essere'],
  pt: ['o', 'a', 'os', 'as', 'que', 'é', 'uma', 'para', 'com', 'não', 'do', 'da', 'como', 'mas', 'também', 'este', 'esta', 'são', 'você', 'pode'],
  pl: ['i', 'nie', 'jest', 'się', 'na', 'to', 'że', 'jak', 'ale', 'dla', 'oraz', 'przez', 'czy', 'tym', 'jego', 'można', 'jeśli', 'tego', 'które', 'także'],
  nl: ['de', 'het', 'een', 'en', 'van', 'is', 'niet', 'dat', 'met', 'voor', 'ook', 'zijn', 'maar', 'als', 'wordt', 'deze', 'kan', 'naar', 'bij', 'dan'],
  cs: ['a', 'je', 'se', 'na', 'to', 'že', 'jak', 'ale', 'pro', 'nebo', 'jsou', 'může', 'také', 'tento', 'když', 'jeho', 'které', 'být', 'aby', 'ještě'],
  tr: ['ve', 'bir', 'bu', 'için', 'ile', 'de', 'da', 'ama', 'gibi', 'daha', 'var', 'olarak', 'çok', 'ne', 'her', 'kadar', 'sonra', 'değil', 'olan', 'ise'],
  sv: ['och', 'att', 'det', 'är', 'en', 'som', 'för', 'inte', 'med', 'till', 'den', 'kan', 'har', 'ett', 'men', 'också', 'eller', 'från', 'när', 'vara'],
  uk: ['і', 'та', 'що', 'це', 'не', 'як', 'для', 'він', 'вона', 'але', 'або', 'також', 'тільки', 'вже', 'якщо', 'його', 'цей', 'ця', 'бути', 'коли'],
  ru: ['и', 'что', 'это', 'не', 'как', 'для', 'он', 'она', 'но', 'или', 'также', 'только', 'уже', 'если', 'его', 'этот', 'эта', 'быть', 'когда', 'чтобы'],
};

const LATIN_LANGUAGES = ['en', 'de', 'fr', 'es', 'it', 'pt', 'pl', 'nl', 'cs', 'tr', 'sv'];
const CYRILLIC_LANGUAGES = ['uk', 'ru'];

const countMatches = (text, pattern) => {
  const matches = text.match(pattern);
  return matches ? matches.length : 0;
};

const scoreStopwords = (words, languages) => {
  const scores = {};
  for (const language of languages) {
    const list = new Set(STOPWORDS[language]);
    let hits = 0;
    for (const word of words) {
      if (list.has(word)) hits += 1;
    }
    scores[language] = hits;
  }
  return scores;
};

const bestOf = (scores, fallback) => {
  let best = fallback;
  let bestScore = 0;
  for (const [language, score] of Object.entries(scores)) {
    if (score > bestScore) {
      best = language;
      bestScore = score;
    }
  }
  return best;
};

const pickByMarkers = (text, markers) => {
  for (const [language, pattern] of Object.entries(markers)) {
    if (pattern.test(text)) return language;
  }
  return null;
};

/**
 * @param {string} text
 * @returns {{ language: string, script: string }} BCP-47 primary language subtag and the dominant script.
 */
export function detectTextLanguage(text) {
  const source = typeof text === 'string' ? text : '';
  const counts = SCRIPT_RANGES.map(([script, pattern]) => [script, countMatches(source, pattern)]);
  const letters = counts.reduce((sum, [, count]) => sum + count, 0);
  if (letters === 0) return { language: 'en', script: 'latin' };

  // Kana settles Japanese even when Han dominates the character count.
  const kana = counts.find(([script]) => script === 'kana')?.[1] ?? 0;
  const han = counts.find(([script]) => script === 'han')?.[1] ?? 0;
  if (kana > 0 && kana + han >= letters * 0.3) return { language: 'ja', script: 'kana' };
  if (han > 0 && han >= letters * 0.3) return { language: 'zh', script: 'han' };

  const [script] = counts.reduce((best, entry) => (entry[1] > best[1] ? entry : best));

  if (script in SCRIPT_LANGUAGE) return { language: SCRIPT_LANGUAGE[script], script };

  const words = source.toLowerCase().split(/[^\p{L}\p{M}']+/u).filter(Boolean);

  if (script === 'cyrillic') {
    const scores = scoreStopwords(words, CYRILLIC_LANGUAGES);
    const ukMarkers = countMatches(source, /[іїєґ]/gi);
    const ruMarkers = countMatches(source, /[ыэъё]/gi);
    // Letters decide: the two alphabets differ in letters that occur in
    // nearly every sentence. Function words only settle a text that shows
    // neither set, and a text with no Russian-only letters is far more
    // likely Ukrainian than the reverse, so that tie goes to Ukrainian.
    if (ukMarkers !== ruMarkers) return { language: ukMarkers > ruMarkers ? 'uk' : 'ru', script };
    if (scores.uk !== scores.ru) return { language: scores.uk > scores.ru ? 'uk' : 'ru', script };
    return { language: ruMarkers > 0 ? 'ru' : 'uk', script };
  }

  const scores = scoreStopwords(words, LATIN_LANGUAGES);
  const marked = pickByMarkers(source, LATIN_MARKERS);
  // A characteristic letter outranks stopword counts unless another language
  // clearly dominates the function words (a German text quoting "façade").
  if (marked && scores[marked] * 2 >= scores[bestOf(scores, marked)]) {
    return { language: marked, script };
  }
  return { language: bestOf(scores, 'en'), script };
}

/**
 * Map a detected language onto the locales a voice list uses (`uk_UA`,
 * `en_US`...). Returns the preferred locale prefixes in order.
 * @param {string} language
 * @returns {string[]}
 */
function localePrefixesForLanguage(language) {
  const table = {
    en: ['en_US', 'en_GB', 'en'],
    uk: ['uk_UA', 'uk'],
    ru: ['ru_RU', 'ru'],
    de: ['de_DE', 'de'],
    fr: ['fr_FR', 'fr_CA', 'fr'],
    es: ['es_ES', 'es_MX', 'es'],
    it: ['it_IT', 'it'],
    pt: ['pt_BR', 'pt_PT', 'pt'],
    pl: ['pl_PL', 'pl'],
    nl: ['nl_NL', 'nl_BE', 'nl'],
    cs: ['cs_CZ', 'cs'],
    tr: ['tr_TR', 'tr'],
    sv: ['sv_SE', 'sv'],
    zh: ['zh_CN', 'zh_TW', 'zh_HK', 'zh'],
    ja: ['ja_JP', 'ja'],
    ko: ['ko_KR', 'ko'],
    el: ['el_GR', 'el'],
    ar: ['ar_001', 'ar_SA', 'ar'],
    he: ['he_IL', 'he'],
    th: ['th_TH', 'th'],
    hi: ['hi_IN', 'hi'],
  };
  return table[language] ?? [language];
}

/**
 * Choose a voice for a language from a `say`-style voice list.
 * Prefers an enhanced/premium variant of a matching voice, then any voice of
 * the exact locale, then any voice of the language. Returns null when the
 * list has no voice for that language.
 * @param {string} language
 * @param {ReadonlyArray<{ name: string, locale: string }>} voices
 * @returns {string | null}
 */
export function pickVoiceForLanguage(language, voices) {
  const prefixes = localePrefixesForLanguage(language);
  for (const prefix of prefixes) {
    const matching = voices.filter((voice) => voice.locale === prefix || voice.locale.startsWith(`${prefix}_`) || (prefix === language && voice.locale.startsWith(`${language}_`)));
    if (matching.length === 0) continue;
    const enhanced = matching.find((voice) => /\((Enhanced|Premium)\)/i.test(voice.name));
    return (enhanced ?? matching[0]).name;
  }
  return null;
}

/**
 * Language of a voice, from its locale (`uk_UA` → `uk`).
 * @param {string | null | undefined} locale
 * @returns {string | null}
 */
export function languageOfLocale(locale) {
  if (typeof locale !== 'string' || !locale) return null;
  return locale.split(/[_-]/)[0].toLowerCase();
}