diff --git a/packages/ui/src/components/chat/markdown/markdown-shiki.worker.ts b/packages/ui/src/components/chat/markdown/markdown-shiki.worker.ts index 98c11361..a06b81fc 100644 --- a/packages/ui/src/components/chat/markdown/markdown-shiki.worker.ts +++ b/packages/ui/src/components/chat/markdown/markdown-shiki.worker.ts @@ -51,6 +51,10 @@ self.onmessage = (event: MessageEvent) => { queue = queue.then(() => highlight(request)).catch(() => {}); return; } + if (request.type === 'highlightTokens') { + queue = queue.then(() => highlightTokens(request)).catch(() => {}); + return; + } queue = queue.then(() => highlightLines(request)).catch(() => {}); }; @@ -83,6 +87,27 @@ async function highlight(request: Extract): Promise { + try { + const instance = await ensureHighlighter(); + if (request.theme && !instance.getLoadedThemes().includes(request.themeName)) { + // Cast: a resolved TextMate theme object from the app theme registry. + await instance.loadTheme(request.theme as Parameters[0]); + } + const lang = await resolveLanguage(instance, request.lang); + const { tokens } = instance.codeToTokens(request.code, { + lang: lang as BundledLanguage, + theme: request.themeName, + }); + const lines = tokens.map((line) => + line.map((token) => [token.content.length, token.color ?? '', token.fontStyle ?? 0] as [number, string, number]), + ); + post({ type: 'highlightTokens', id: request.id, lines }); + } catch (error) { + post({ type: 'error', id: request.id, message: error instanceof Error ? error.message : String(error) }); + } +} + async function highlightLines(request: Extract): Promise { try { const instance = await ensureHighlighter(); diff --git a/packages/ui/src/components/chat/markdown/markdown-worker-protocol.ts b/packages/ui/src/components/chat/markdown/markdown-worker-protocol.ts index 1aec6cfe..ca52788e 100644 --- a/packages/ui/src/components/chat/markdown/markdown-worker-protocol.ts +++ b/packages/ui/src/components/chat/markdown/markdown-worker-protocol.ts @@ -4,6 +4,11 @@ // ready-to-splice Shiki HTML. The theme is dependency-free and imported inside // the worker directly, so it is not sent over postMessage. +// A single styled run inside a line: [length, color, fontStyleBits]. +// `color` is '' for default-foreground runs; fontStyleBits is Shiki's FontStyle +// bitmask (1=italic, 2=bold, 4=underline). +export type MarkdownTokenRun = [length: number, color: string, fontStyle: number]; + export type MarkdownWorkerRequest = | { type: 'init' } // Highlight a whole block to ready-to-splice Shiki `
` HTML.
@@ -11,9 +16,15 @@ export type MarkdownWorkerRequest =
   // Highlight a whole block but return per-line inner HTML (one entry per line),
   // so per-line layouts (diffs, gutters, virtualization) tokenize in ONE call
   // instead of one worker round-trip per line.
-  | { type: 'highlightLines'; id: number; code: string; lang: string };
+  | { type: 'highlightLines'; id: number; code: string; lang: string }
+  // Tokenize with an arbitrary registered theme and return per-line styled runs
+  // with offsets — for building CodeMirror decorations. `theme` (a resolved
+  // TextMate theme object) is sent only the first time a theme name is used;
+  // afterwards only `themeName` is sent and the worker reuses the loaded theme.
+  | { type: 'highlightTokens'; id: number; code: string; lang: string; themeName: string; theme?: unknown };
 
 export type MarkdownWorkerResponse =
   | { type: 'highlight'; id: number; html: string }
   | { type: 'highlightLines'; id: number; lines: string[] }
+  | { type: 'highlightTokens'; id: number; lines: MarkdownTokenRun[][] }
   | { type: 'error'; id: number; message: string };
diff --git a/packages/ui/src/components/chat/markdown/markdown-worker.ts b/packages/ui/src/components/chat/markdown/markdown-worker.ts
index 70411201..3de12e32 100644
--- a/packages/ui/src/components/chat/markdown/markdown-worker.ts
+++ b/packages/ui/src/components/chat/markdown/markdown-worker.ts
@@ -1,5 +1,5 @@
 import MarkdownShikiWorkerUrl from './markdown-shiki.worker.ts?worker&url';
-import type { MarkdownWorkerRequest, MarkdownWorkerResponse } from './markdown-worker-protocol';
+import type { MarkdownTokenRun, MarkdownWorkerRequest, MarkdownWorkerResponse } from './markdown-worker-protocol';
 
 // Main-thread client for the markdown Shiki worker. Moves syntax tokenization
 // off the UI thread: a closed code block is shipped to the worker, which returns
@@ -12,10 +12,14 @@ type PendingResolver = (response: MarkdownWorkerResponse | null) => void;
 let worker: Worker | undefined;
 let nextId = 0;
 const pending = new Map();
+// Theme names whose full definition we've already shipped to the live worker, so
+// repeat tokenization sends only the name (not the whole theme object) again.
+const sentThemes = new Set();
 
 const failAll = (): void => {
   pending.forEach((resolve) => resolve(null));
   pending.clear();
+  sentThemes.clear();
   worker?.terminate();
   worker = undefined;
 };
@@ -68,3 +72,31 @@ export const highlightLinesInWorker = async (code: string, lang: string): Promis
   const response = await request((id) => ({ type: 'highlightLines', id, code, lang }));
   return response?.type === 'highlightLines' ? response.lines : null;
 };
+
+/**
+ * Tokenize `code` with the given resolved TextMate theme and return per-line
+ * styled runs with offsets — for building CodeMirror decorations that match the
+ * Shiki file view exactly. The full theme object is shipped only the first time
+ * a theme name is seen by the live worker. Resolves to `null` on failure.
+ */
+export const highlightTokensInWorker = async (
+  code: string,
+  lang: string,
+  themeName: string,
+  theme: unknown,
+): Promise => {
+  const needsTheme = !sentThemes.has(themeName);
+  const response = await request((id) => ({
+    type: 'highlightTokens',
+    id,
+    code,
+    lang,
+    themeName,
+    ...(needsTheme ? { theme } : {}),
+  }));
+  if (response?.type === 'highlightTokens') {
+    sentThemes.add(themeName);
+    return response.lines;
+  }
+  return null;
+};
diff --git a/packages/ui/src/components/views/FilesView.tsx b/packages/ui/src/components/views/FilesView.tsx
index 8dc39d0a..3bd45c44 100644
--- a/packages/ui/src/components/views/FilesView.tsx
+++ b/packages/ui/src/components/views/FilesView.tsx
@@ -29,6 +29,8 @@ import { JsonTreeView } from '@/components/ui/JsonTreeView';
 import { SimpleMarkdownRenderer } from '@/components/chat/MarkdownRenderer';
 import { languageByExtension, loadLanguageByExtension } from '@/lib/codemirror/languageByExtension';
 import { createFlexokiCodeMirrorTheme } from '@/lib/codemirror/flexokiTheme';
+import { shikiHighlightExtension } from '@/lib/codemirror/shikiHighlight';
+import { getResolvedShikiTheme } from '@/lib/shiki/appThemeRegistry';
 import { File as PierreFile } from '@pierre/diffs/react';
 import {
   Dialog,
@@ -2824,11 +2826,23 @@ export const FilesView: React.FC = ({ mode = 'full' }) => {
       return [createFlexokiCodeMirrorTheme(currentTheme)];
     }
 
-    const extensions = [createFlexokiCodeMirrorTheme(currentTheme)];
+    // Shiki token colors (worker-backed) match the Shiki file view exactly.
+    // Same language resolver as the view, so both agree on the language. When
+    // Shiki is the color source, drop the lezer token colors to avoid a
+    // competing highlighter (keep the lezer language for indentation/folding).
+    const shikiLanguage = getLanguageFromExtension(selectedFile.path);
+    const extensions = [createFlexokiCodeMirrorTheme(currentTheme, shikiLanguage ? { syntaxColors: false } : undefined)];
     const language = staticLanguageExtension ?? dynamicLanguageExtension;
     if (language) {
       extensions.push(language);
     }
+    if (shikiLanguage) {
+      extensions.push(shikiHighlightExtension({
+        language: shikiLanguage,
+        themeName: currentTheme.metadata.id,
+        theme: getResolvedShikiTheme(currentTheme),
+      }));
+    }
     if (wrapLines) {
       extensions.push(EditorView.lineWrapping);
     }
@@ -2939,7 +2953,7 @@ export const FilesView: React.FC = ({ mode = 'full' }) => {
   }, [pdfAssetAuthKey, t]);
 
   const isPdfAssetAuthLoading = Boolean(pdfAssetAuthKey && pdfAssetAuthReadyKey !== pdfAssetAuthKey);
-
+
   const imageSrc = selectedFile?.path && isSelectedImage
     ? (runtime.isDesktop
       ? (isSelectedSvg
diff --git a/packages/ui/src/lib/codemirror/flexokiTheme.ts b/packages/ui/src/lib/codemirror/flexokiTheme.ts
index 8fb2bd6c..f1acf0c5 100644
--- a/packages/ui/src/lib/codemirror/flexokiTheme.ts
+++ b/packages/ui/src/lib/codemirror/flexokiTheme.ts
@@ -6,7 +6,13 @@ import { classHighlighter, tags as t } from '@lezer/highlight';
 
 import type { Theme } from '@/types/theme';
 
-export function createFlexokiCodeMirrorTheme(theme: Theme): Extension {
+export function createFlexokiCodeMirrorTheme(
+  theme: Theme,
+  // When syntax colors are provided elsewhere (e.g. the Shiki decoration
+  // extension), pass `{ syntaxColors: false }` to keep only the editor UI theme
+  // (gutters, selection, cursor) and avoid a competing token color source.
+  options?: { syntaxColors?: boolean },
+): Extension {
   const isDark = theme.metadata.variant === 'dark';
 
   const monoFont = theme.config?.fonts?.mono || 'monospace';
@@ -554,6 +560,10 @@ export function createFlexokiCodeMirrorTheme(theme: Theme): Extension {
     },
   ]);
 
+  if (options?.syntaxColors === false) {
+    return [ui];
+  }
+
   return [
     ui,
     syntaxHighlighting(defaultHighlightStyle, { fallback: true }),
diff --git a/packages/ui/src/lib/codemirror/shikiHighlight.ts b/packages/ui/src/lib/codemirror/shikiHighlight.ts
new file mode 100644
index 00000000..07941453
--- /dev/null
+++ b/packages/ui/src/lib/codemirror/shikiHighlight.ts
@@ -0,0 +1,135 @@
+import type { Extension } from '@codemirror/state';
+import { RangeSetBuilder, StateEffect, StateField } from '@codemirror/state';
+import { Decoration, type DecorationSet, EditorView, ViewPlugin, type ViewUpdate } from '@codemirror/view';
+
+import { highlightTokensInWorker } from '@/components/chat/markdown/markdown-worker';
+import type { MarkdownTokenRun } from '@/components/chat/markdown/markdown-worker-protocol';
+
+// Shiki-powered syntax highlighting for CodeMirror, matching the Shiki file
+// view exactly: tokenize the whole document in the markdown Shiki worker (off
+// the main thread) and project the tokens onto mark decorations.
+//
+// While typing, existing decorations are mapped through edits so colors stay put
+// (no flash); the document is re-tokenized on a short idle, never on the hot
+// keystroke path. Decoration inline styles override the lezer HighlightStyle, so
+// the lezer language extension can stay on for indentation/folding/brackets.
+
+const RETOKENIZE_IDLE_MS = 180;
+
+const setShikiDecorations = StateEffect.define();
+
+// Shiki FontStyle bitmask.
+const FONT_STYLE_ITALIC = 1;
+const FONT_STYLE_BOLD = 2;
+const FONT_STYLE_UNDERLINE = 4;
+
+// Mark decorations are interned by color+style so repeated tokens reuse one spec.
+const markCache = new Map();
+
+const markFor = (color: string, fontStyle: number): Decoration => {
+  const key = `${color}|${fontStyle}`;
+  const cached = markCache.get(key);
+  if (cached) return cached;
+  let style = '';
+  if (color) style += `color:${color};`;
+  if (fontStyle & FONT_STYLE_ITALIC) style += 'font-style:italic;';
+  if (fontStyle & FONT_STYLE_BOLD) style += 'font-weight:bold;';
+  if (fontStyle & FONT_STYLE_UNDERLINE) style += 'text-decoration:underline;';
+  const decoration = Decoration.mark({ attributes: { style } });
+  markCache.set(key, decoration);
+  return decoration;
+};
+
+const buildDecorations = (view: EditorView, lines: MarkdownTokenRun[][]): DecorationSet => {
+  const builder = new RangeSetBuilder();
+  const doc = view.state.doc;
+  const count = Math.min(lines.length, doc.lines);
+  for (let i = 0; i < count; i += 1) {
+    const runs = lines[i];
+    if (!runs || runs.length === 0) continue;
+    const line = doc.line(i + 1);
+    let pos = line.from;
+    for (const [length, color, fontStyle] of runs) {
+      if (length <= 0) continue;
+      const from = pos;
+      const to = Math.min(pos + length, line.to);
+      pos += length;
+      if (to <= from) continue;
+      if (!color && !fontStyle) continue;
+      builder.add(from, to, markFor(color, fontStyle));
+    }
+  }
+  return builder.finish();
+};
+
+const shikiDecorationsField = StateField.define({
+  create() {
+    return Decoration.none;
+  },
+  update(decorations, tr) {
+    // Map existing colors through edits so they track the text while typing.
+    let next = decorations.map(tr.changes);
+    for (const effect of tr.effects) {
+      if (effect.is(setShikiDecorations)) next = effect.value;
+    }
+    return next;
+  },
+  provide: (field) => EditorView.decorations.from(field),
+});
+
+type ShikiHighlightOptions = {
+  /** Shiki language id (e.g. 'typescript'). */
+  language: string;
+  /** App theme id — the registered TextMate theme name to tokenize with. */
+  themeName: string;
+  /** Resolved TextMate theme object (shipped to the worker once per name). */
+  theme: unknown;
+};
+
+const shikiHighlightPlugin = (options: ShikiHighlightOptions) =>
+  ViewPlugin.fromClass(
+    class {
+      private timer: ReturnType | undefined;
+      private generation = 0;
+
+      constructor(view: EditorView) {
+        void this.tokenize(view);
+      }
+
+      update(update: ViewUpdate) {
+        if (update.docChanged) this.schedule(update.view);
+      }
+
+      private schedule(view: EditorView) {
+        if (this.timer) clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+          this.timer = undefined;
+          void this.tokenize(view);
+        }, RETOKENIZE_IDLE_MS);
+      }
+
+      private async tokenize(view: EditorView) {
+        const generation = ++this.generation;
+        const text = view.state.doc.toString();
+        const lines = await highlightTokensInWorker(text, options.language, options.themeName, options.theme);
+        if (!lines) return;
+        // Drop if a newer tokenization started or the doc length changed since.
+        if (generation !== this.generation) return;
+        if (view.state.doc.length !== text.length) return;
+        view.dispatch({ effects: setShikiDecorations.of(buildDecorations(view, lines)) });
+      }
+
+      destroy() {
+        if (this.timer) clearTimeout(this.timer);
+      }
+    },
+  );
+
+/**
+ * CodeMirror extension that colors the document with Shiki tokens via the
+ * worker. Recreate it (new instance) when the language or theme changes.
+ */
+export const shikiHighlightExtension = (options: ShikiHighlightOptions): Extension => [
+  shikiDecorationsField,
+  shikiHighlightPlugin(options),
+];