feat(stt): add server-side STT provider via OpenAI-compatible Whisper endpoint (#860)

Co-authored-by: Alexander Busse <alex@ableph.net>
2026-04-11 22:47:33 +03:00
parent 75a10ea66c
commit 3746d99a85
6 changed files with 836 additions and 18 deletions
@@ -1,3 +1,5 @@
+import express from 'express';
+
 export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
  let ttsModulePromise = null;
  const getTtsModule = async () => {
@@ -222,4 +224,54 @@ export function registerTtsRoutes(app, { resolveZenModel, sayTTSCapability }) {
      });
    }
  });
+
+  // Server-side STT: receive raw audio, proxy to OpenAI-compatible transcription endpoint
+  app.post(
+    '/api/stt/transcribe',
+    express.raw({ type: (req) => (req.headers['content-type'] || '').startsWith('audio/'), limit: '20mb' }),
+    async (req, res) => {
+      try {
+        const { transcribeAudio } = await import('./stt.js');
+
+        const mimeType = (req.headers['content-type'] || 'audio/webm').split(',')[0].trim();
+        const baseURL = req.headers['x-base-url'];
+        const model = req.headers['x-model'] || 'deepdml/faster-whisper-large-v3-turbo-ct2';
+        const language = req.headers['x-language'] || undefined;
+
+        if (!req.body || !Buffer.isBuffer(req.body) || req.body.length === 0) {
+          return res.status(400).json({ error: 'Audio data is required' });
+        }
+
+        if (!baseURL) {
+          return res.status(400).json({ error: 'X-Base-URL header is required' });
+        }
+
+        console.log('[STT] Transcribing audio:', {
+          bytes: req.body.length,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        const transcript = await transcribeAudio({
+          audioBuffer: req.body,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        console.log('[STT] Transcript:', transcript?.slice(0, 120));
+        res.json({ transcript: transcript ?? '' });
+      } catch (error) {
+        console.error('[STT] Error:', error);
+        if (!res.headersSent) {
+          res.status(500).json({
+            error: error instanceof Error ? error.message : 'Transcription failed',
+          });
+        }
+      }
+    }
+  );
 }
@@ -0,0 +1,66 @@
+/**
+ * Server-side Speech-to-Text Service
+ *
+ * Proxies audio to any OpenAI-compatible transcription endpoint
+ * (e.g. faster-whisper, whisper.cpp) using the OpenAI Node SDK.
+ */
+
+import OpenAI, { toFile } from 'openai';
+
+/**
+ * Transcribe an audio buffer via an OpenAI-compatible /v1/audio/transcriptions endpoint.
+ *
+ * @param {object} opts
+ * @param {Buffer} opts.audioBuffer  - Raw audio bytes
+ * @param {string} opts.mimeType     - MIME type of the audio (e.g. 'audio/webm')
+ * @param {string} opts.model        - Model name accepted by the remote server
+ * @param {string} [opts.baseURL]    - Base URL of the compatible server (including /v1)
+ * @param {string} [opts.language]   - Optional BCP-47 language hint (e.g. 'en')
+ * @returns {Promise<string>} Transcribed text
+ */
+export async function transcribeAudio({ audioBuffer, mimeType, model, baseURL, language }) {
+  const clientOpts = {
+    apiKey: process.env.OPENAI_API_KEY || 'not-required',
+  };
+  if (baseURL) {
+    clientOpts.baseURL = baseURL;
+  }
+
+  const client = new OpenAI(clientOpts);
+
+  // Derive a sensible filename extension from the MIME type so the server
+  // can infer the codec when it isn't explicit in the stream header.
+  const ext = mimeTypeToExt(mimeType);
+  const filename = `audio.${ext}`;
+
+  const file = await toFile(audioBuffer, filename, { type: mimeType });
+
+  const result = await client.audio.transcriptions.create({
+    file,
+    model,
+    response_format: 'json',
+    ...(language ? { language } : {}),
+  });
+
+  return result.text ?? '';
+}
+
+/**
+ * Map a MIME type to a file extension understood by Whisper servers.
+ * @param {string} mimeType
+ * @returns {string}
+ */
+function mimeTypeToExt(mimeType) {
+  const type = (mimeType || '').split(';')[0].trim().toLowerCase();
+  const map = {
+    'audio/webm': 'webm',
+    'audio/ogg': 'ogg',
+    'audio/wav': 'wav',
+    'audio/wave': 'wav',
+    'audio/mpeg': 'mp3',
+    'audio/mp4': 'mp4',
+    'audio/mp3': 'mp3',
+    'audio/flac': 'flac',
+  };
+  return map[type] ?? 'webm';
+}