feat(editor): Shiki syntax highlighting in the file editor (CodeMirror)

Bring the CodeMirror file editor up to the same rich highlighting as the Shiki
file view, so toggling edit <-> view is visually consistent. lezer collapses far
more tokens than TextMate (import/from/const are all "keyword"), so a theme
remap can't reach parity — instead, project real Shiki tokens onto decorations.

- Worker: add highlightTokens — tokenize with an arbitrary registered TextMate
  theme and return per-line styled runs with offsets. The theme object ships to
  the worker once per name; later calls send only the name.
- New shikiHighlight CodeMirror extension: a StateField of mark decorations
  built from worker tokens. Re-tokenizes on a short idle (off the keystroke
  path) and maps decorations through edits so colors persist while typing.
- flexokiTheme: add { syntaxColors: false } to keep only the editor UI theme,
  so the lezer highlighter doesn't compete with the Shiki decorations.
- FilesView: enable Shiki highlighting (same language resolver as the file view
  → identical language) and drop lezer token colors when it's active. lezer
  language stays on for indentation/folding/brackets.

packages/ui/src/components/chat/markdown/markdown-shiki.worker.ts

@@ -51,6 +51,10 @@ self.onmessage = (event: MessageEvent<MarkdownWorkerRequest>) => {
     queue = queue.then(() => highlight(request)).catch(() => {});
     return;
   }
+  if (request.type === 'highlightTokens') {
+    queue = queue.then(() => highlightTokens(request)).catch(() => {});
+    return;
+  }
   queue = queue.then(() => highlightLines(request)).catch(() => {});
 };
 
@@ -83,6 +87,27 @@ async function highlight(request: Extract<MarkdownWorkerRequest, { type: 'highli
   }
 }
 
+async function highlightTokens(request: Extract<MarkdownWorkerRequest, { type: 'highlightTokens' }>): Promise<void> {
+  try {
+    const instance = await ensureHighlighter();
+    if (request.theme && !instance.getLoadedThemes().includes(request.themeName)) {
+      // Cast: a resolved TextMate theme object from the app theme registry.
+      await instance.loadTheme(request.theme as Parameters<typeof instance.loadTheme>[0]);
+    }
+    const lang = await resolveLanguage(instance, request.lang);
+    const { tokens } = instance.codeToTokens(request.code, {
+      lang: lang as BundledLanguage,
+      theme: request.themeName,
+    });
+    const lines = tokens.map((line) =>
+      line.map((token) => [token.content.length, token.color ?? '', token.fontStyle ?? 0] as [number, string, number]),
+    );
+    post({ type: 'highlightTokens', id: request.id, lines });
+  } catch (error) {
+    post({ type: 'error', id: request.id, message: error instanceof Error ? error.message : String(error) });
+  }
+}
+
 async function highlightLines(request: Extract<MarkdownWorkerRequest, { type: 'highlightLines' }>): Promise<void> {
   try {
     const instance = await ensureHighlighter();

packages/ui/src/components/chat/markdown/markdown-worker-protocol.ts

@@ -4,6 +4,11 @@
 // ready-to-splice Shiki HTML. The theme is dependency-free and imported inside
 // the worker directly, so it is not sent over postMessage.
 
+// A single styled run inside a line: [length, color, fontStyleBits].
+// `color` is '' for default-foreground runs; fontStyleBits is Shiki's FontStyle
+// bitmask (1=italic, 2=bold, 4=underline).
+export type MarkdownTokenRun = [length: number, color: string, fontStyle: number];
+
 export type MarkdownWorkerRequest =
   | { type: 'init' }
   // Highlight a whole block to ready-to-splice Shiki `<pre>` HTML.
@@ -11,9 +16,15 @@ export type MarkdownWorkerRequest =
   // Highlight a whole block but return per-line inner HTML (one entry per line),
   // so per-line layouts (diffs, gutters, virtualization) tokenize in ONE call
   // instead of one worker round-trip per line.
-  | { type: 'highlightLines'; id: number; code: string; lang: string };
+  | { type: 'highlightLines'; id: number; code: string; lang: string }
+  // Tokenize with an arbitrary registered theme and return per-line styled runs
+  // with offsets — for building CodeMirror decorations. `theme` (a resolved
+  // TextMate theme object) is sent only the first time a theme name is used;
+  // afterwards only `themeName` is sent and the worker reuses the loaded theme.
+  | { type: 'highlightTokens'; id: number; code: string; lang: string; themeName: string; theme?: unknown };
 
 export type MarkdownWorkerResponse =
   | { type: 'highlight'; id: number; html: string }
   | { type: 'highlightLines'; id: number; lines: string[] }
+  | { type: 'highlightTokens'; id: number; lines: MarkdownTokenRun[][] }
   | { type: 'error'; id: number; message: string };

packages/ui/src/components/chat/markdown/markdown-worker.ts

@@ -1,5 +1,5 @@
 import MarkdownShikiWorkerUrl from './markdown-shiki.worker.ts?worker&url';
-import type { MarkdownWorkerRequest, MarkdownWorkerResponse } from './markdown-worker-protocol';
+import type { MarkdownTokenRun, MarkdownWorkerRequest, MarkdownWorkerResponse } from './markdown-worker-protocol';
 
 // Main-thread client for the markdown Shiki worker. Moves syntax tokenization
 // off the UI thread: a closed code block is shipped to the worker, which returns
@@ -12,10 +12,14 @@ type PendingResolver = (response: MarkdownWorkerResponse | null) => void;
 let worker: Worker | undefined;
 let nextId = 0;
 const pending = new Map<number, PendingResolver>();
+// Theme names whose full definition we've already shipped to the live worker, so
+// repeat tokenization sends only the name (not the whole theme object) again.
+const sentThemes = new Set<string>();
 
 const failAll = (): void => {
   pending.forEach((resolve) => resolve(null));
   pending.clear();
+  sentThemes.clear();
   worker?.terminate();
   worker = undefined;
 };
@@ -68,3 +72,31 @@ export const highlightLinesInWorker = async (code: string, lang: string): Promis
   const response = await request((id) => ({ type: 'highlightLines', id, code, lang }));
   return response?.type === 'highlightLines' ? response.lines : null;
 };
+
+/**
+ * Tokenize `code` with the given resolved TextMate theme and return per-line
+ * styled runs with offsets — for building CodeMirror decorations that match the
+ * Shiki file view exactly. The full theme object is shipped only the first time
+ * a theme name is seen by the live worker. Resolves to `null` on failure.
+ */
+export const highlightTokensInWorker = async (
+  code: string,
+  lang: string,
+  themeName: string,
+  theme: unknown,
+): Promise<MarkdownTokenRun[][] | null> => {
+  const needsTheme = !sentThemes.has(themeName);
+  const response = await request((id) => ({
+    type: 'highlightTokens',
+    id,
+    code,
+    lang,
+    themeName,
+    ...(needsTheme ? { theme } : {}),
+  }));
+  if (response?.type === 'highlightTokens') {
+    sentThemes.add(themeName);
+    return response.lines;
+  }
+  return null;
+};

packages/ui/src/components/views/FilesView.tsx

@@ -29,6 +29,8 @@ import { JsonTreeView } from '@/components/ui/JsonTreeView';
 import { SimpleMarkdownRenderer } from '@/components/chat/MarkdownRenderer';
 import { languageByExtension, loadLanguageByExtension } from '@/lib/codemirror/languageByExtension';
 import { createFlexokiCodeMirrorTheme } from '@/lib/codemirror/flexokiTheme';
+import { shikiHighlightExtension } from '@/lib/codemirror/shikiHighlight';
+import { getResolvedShikiTheme } from '@/lib/shiki/appThemeRegistry';
 import { File as PierreFile } from '@pierre/diffs/react';
 import {
   Dialog,
@@ -2824,11 +2826,23 @@ export const FilesView: React.FC<FilesViewProps> = ({ mode = 'full' }) => {
       return [createFlexokiCodeMirrorTheme(currentTheme)];
     }
 
-    const extensions = [createFlexokiCodeMirrorTheme(currentTheme)];
+    // Shiki token colors (worker-backed) match the Shiki file view exactly.
+    // Same language resolver as the view, so both agree on the language. When
+    // Shiki is the color source, drop the lezer token colors to avoid a
+    // competing highlighter (keep the lezer language for indentation/folding).
+    const shikiLanguage = getLanguageFromExtension(selectedFile.path);
+    const extensions = [createFlexokiCodeMirrorTheme(currentTheme, shikiLanguage ? { syntaxColors: false } : undefined)];
     const language = staticLanguageExtension ?? dynamicLanguageExtension;
     if (language) {
       extensions.push(language);
     }
+    if (shikiLanguage) {
+      extensions.push(shikiHighlightExtension({
+        language: shikiLanguage,
+        themeName: currentTheme.metadata.id,
+        theme: getResolvedShikiTheme(currentTheme),
+      }));
+    }
     if (wrapLines) {
       extensions.push(EditorView.lineWrapping);
     }
@@ -2939,7 +2953,7 @@ export const FilesView: React.FC<FilesViewProps> = ({ mode = 'full' }) => {
   }, [pdfAssetAuthKey, t]);
 
   const isPdfAssetAuthLoading = Boolean(pdfAssetAuthKey && pdfAssetAuthReadyKey !== pdfAssetAuthKey);
-
+
   const imageSrc = selectedFile?.path && isSelectedImage
     ? (runtime.isDesktop
       ? (isSelectedSvg

packages/ui/src/lib/codemirror/flexokiTheme.ts

@@ -6,7 +6,13 @@ import { classHighlighter, tags as t } from '@lezer/highlight';
 
 import type { Theme } from '@/types/theme';
 
-export function createFlexokiCodeMirrorTheme(theme: Theme): Extension {
+export function createFlexokiCodeMirrorTheme(
+  theme: Theme,
+  // When syntax colors are provided elsewhere (e.g. the Shiki decoration
+  // extension), pass `{ syntaxColors: false }` to keep only the editor UI theme
+  // (gutters, selection, cursor) and avoid a competing token color source.
+  options?: { syntaxColors?: boolean },
+): Extension {
   const isDark = theme.metadata.variant === 'dark';
 
   const monoFont = theme.config?.fonts?.mono || 'monospace';
@@ -554,6 +560,10 @@ export function createFlexokiCodeMirrorTheme(theme: Theme): Extension {
     },
   ]);
 
+  if (options?.syntaxColors === false) {
+    return [ui];
+  }
+
   return [
     ui,
     syntaxHighlighting(defaultHighlightStyle, { fallback: true }),

packages/ui/src/lib/codemirror/shikiHighlight.ts

@@ -0,0 +1,135 @@
+import type { Extension } from '@codemirror/state';
+import { RangeSetBuilder, StateEffect, StateField } from '@codemirror/state';
+import { Decoration, type DecorationSet, EditorView, ViewPlugin, type ViewUpdate } from '@codemirror/view';
+
+import { highlightTokensInWorker } from '@/components/chat/markdown/markdown-worker';
+import type { MarkdownTokenRun } from '@/components/chat/markdown/markdown-worker-protocol';
+
+// Shiki-powered syntax highlighting for CodeMirror, matching the Shiki file
+// view exactly: tokenize the whole document in the markdown Shiki worker (off
+// the main thread) and project the tokens onto mark decorations.
+//
+// While typing, existing decorations are mapped through edits so colors stay put
+// (no flash); the document is re-tokenized on a short idle, never on the hot
+// keystroke path. Decoration inline styles override the lezer HighlightStyle, so
+// the lezer language extension can stay on for indentation/folding/brackets.
+
+const RETOKENIZE_IDLE_MS = 180;
+
+const setShikiDecorations = StateEffect.define<DecorationSet>();
+
+// Shiki FontStyle bitmask.
+const FONT_STYLE_ITALIC = 1;
+const FONT_STYLE_BOLD = 2;
+const FONT_STYLE_UNDERLINE = 4;
+
+// Mark decorations are interned by color+style so repeated tokens reuse one spec.
+const markCache = new Map<string, Decoration>();
+
+const markFor = (color: string, fontStyle: number): Decoration => {
+  const key = `${color}|${fontStyle}`;
+  const cached = markCache.get(key);
+  if (cached) return cached;
+  let style = '';
+  if (color) style += `color:${color};`;
+  if (fontStyle & FONT_STYLE_ITALIC) style += 'font-style:italic;';
+  if (fontStyle & FONT_STYLE_BOLD) style += 'font-weight:bold;';
+  if (fontStyle & FONT_STYLE_UNDERLINE) style += 'text-decoration:underline;';
+  const decoration = Decoration.mark({ attributes: { style } });
+  markCache.set(key, decoration);
+  return decoration;
+};
+
+const buildDecorations = (view: EditorView, lines: MarkdownTokenRun[][]): DecorationSet => {
+  const builder = new RangeSetBuilder<Decoration>();
+  const doc = view.state.doc;
+  const count = Math.min(lines.length, doc.lines);
+  for (let i = 0; i < count; i += 1) {
+    const runs = lines[i];
+    if (!runs || runs.length === 0) continue;
+    const line = doc.line(i + 1);
+    let pos = line.from;
+    for (const [length, color, fontStyle] of runs) {
+      if (length <= 0) continue;
+      const from = pos;
+      const to = Math.min(pos + length, line.to);
+      pos += length;
+      if (to <= from) continue;
+      if (!color && !fontStyle) continue;
+      builder.add(from, to, markFor(color, fontStyle));
+    }
+  }
+  return builder.finish();
+};
+
+const shikiDecorationsField = StateField.define<DecorationSet>({
+  create() {
+    return Decoration.none;
+  },
+  update(decorations, tr) {
+    // Map existing colors through edits so they track the text while typing.
+    let next = decorations.map(tr.changes);
+    for (const effect of tr.effects) {
+      if (effect.is(setShikiDecorations)) next = effect.value;
+    }
+    return next;
+  },
+  provide: (field) => EditorView.decorations.from(field),
+});
+
+type ShikiHighlightOptions = {
+  /** Shiki language id (e.g. 'typescript'). */
+  language: string;
+  /** App theme id — the registered TextMate theme name to tokenize with. */
+  themeName: string;
+  /** Resolved TextMate theme object (shipped to the worker once per name). */
+  theme: unknown;
+};
+
+const shikiHighlightPlugin = (options: ShikiHighlightOptions) =>
+  ViewPlugin.fromClass(
+    class {
+      private timer: ReturnType<typeof setTimeout> | undefined;
+      private generation = 0;
+
+      constructor(view: EditorView) {
+        void this.tokenize(view);
+      }
+
+      update(update: ViewUpdate) {
+        if (update.docChanged) this.schedule(update.view);
+      }
+
+      private schedule(view: EditorView) {
+        if (this.timer) clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+          this.timer = undefined;
+          void this.tokenize(view);
+        }, RETOKENIZE_IDLE_MS);
+      }
+
+      private async tokenize(view: EditorView) {
+        const generation = ++this.generation;
+        const text = view.state.doc.toString();
+        const lines = await highlightTokensInWorker(text, options.language, options.themeName, options.theme);
+        if (!lines) return;
+        // Drop if a newer tokenization started or the doc length changed since.
+        if (generation !== this.generation) return;
+        if (view.state.doc.length !== text.length) return;
+        view.dispatch({ effects: setShikiDecorations.of(buildDecorations(view, lines)) });
+      }
+
+      destroy() {
+        if (this.timer) clearTimeout(this.timer);
+      }
+    },
+  );
+
+/**
+ * CodeMirror extension that colors the document with Shiki tokens via the
+ * worker. Recreate it (new instance) when the language or theme changes.
+ */
+export const shikiHighlightExtension = (options: ShikiHighlightOptions): Extension => [
+  shikiDecorationsField,
+  shikiHighlightPlugin(options),
+];