From f790b58d83bcaea7735ae5445a3ab5743f4cc99a Mon Sep 17 00:00:00 2001 From: ChangeHow Date: Thu, 13 Aug 2026 14:32:08 +0800 Subject: [PATCH 1/4] feat: add markdown image gallery previews --- .../components/chat/MarkdownImageGallery.tsx | 115 ++++++++++ .../src/components/chat/MarkdownRenderer.tsx | 10 + .../components/chat/MarkdownRendererImpl.tsx | 95 ++++++++- .../chat/markdown/markdownCore.test.ts | 115 +++++++++- .../components/chat/markdown/markdownCore.ts | 117 ++++++++++- .../chat/markdown/markdownImageAssets.ts | 132 ++++++++++++ .../components/chat/message/MessageBody.tsx | 13 +- .../chat/message/ToolOutputDialog.test.ts | 41 ++++ .../chat/message/ToolOutputDialog.tsx | 198 ++++++++---------- .../chat/message/imagePreviewSizing.ts | 36 ++++ .../chat/message/parts/AssistantTextPart.tsx | 3 + .../chat/message/parts/DOCUMENTATION.md | 13 ++ packages/ui/src/index.css | 6 + packages/vscode/src/DOCUMENTATION.md | 1 + .../vscode/src/bridge-fs-helpers-runtime.ts | 12 +- .../src/bridge-localfs-proxy-runtime.test.js | 61 +++--- .../src/bridge-localfs-proxy-runtime.ts | 5 +- 17 files changed, 818 insertions(+), 155 deletions(-) create mode 100644 packages/ui/src/components/chat/MarkdownImageGallery.tsx create mode 100644 packages/ui/src/components/chat/markdown/markdownImageAssets.ts create mode 100644 packages/ui/src/components/chat/message/imagePreviewSizing.ts diff --git a/packages/ui/src/components/chat/MarkdownImageGallery.tsx b/packages/ui/src/components/chat/MarkdownImageGallery.tsx new file mode 100644 index 00000000..bdb2aa27 --- /dev/null +++ b/packages/ui/src/components/chat/MarkdownImageGallery.tsx @@ -0,0 +1,115 @@ +import React from 'react'; +import { Icon } from '@/components/icon/Icon'; +import { useEffectiveDirectory } from '@/hooks/useEffectiveDirectory'; +import type { ToolPopupContent } from './message/types'; +import { + extractMarkdownImageCandidates, + MAX_MARKDOWN_IMAGE_COUNT, + type MarkdownImageCandidate, +} from './markdown/markdownCore'; +import { resolveMarkdownImageSource } from './markdown/markdownImageAssets'; + +const MarkdownImageThumbnail: React.FC<{ + candidate: MarkdownImageCandidate; + directory: string; + onShowPopup?: (content: ToolPopupContent) => void; +}> = ({ candidate, directory, onShowPopup }) => { + const [image, setImage] = React.useState<{ + url: string; + status: 'loading' | 'ready' | 'error'; + }>({ url: '', status: 'loading' }); + + React.useEffect(() => { + const controller = new AbortController(); + setImage({ url: '', status: 'loading' }); + void resolveMarkdownImageSource(candidate.source, directory, controller.signal) + .then((url) => { + if (!controller.signal.aborted) setImage({ url, status: 'loading' }); + }) + .catch(() => { + if (!controller.signal.aborted) setImage({ url: '', status: 'error' }); + }); + return () => controller.abort(); + }, [candidate.source, directory]); + + const openPreview = React.useCallback(() => { + if (image.status !== 'ready' || !onShowPopup) return; + onShowPopup({ + open: true, + title: candidate.filename, + content: '', + metadata: { tool: 'markdown-image-preview', filename: candidate.filename }, + image: { url: image.url, filename: candidate.filename }, + }); + }, [candidate.filename, image, onShowPopup]); + + return ( + + ); +}; + +export const MarkdownImageGallery: React.FC<{ + contents: readonly string[]; + onShowPopup?: (content: ToolPopupContent) => void; +}> = ({ contents, onShowPopup }) => { + const directory = useEffectiveDirectory() ?? ''; + const candidates = React.useMemo( + () => extractMarkdownImageCandidates(contents, MAX_MARKDOWN_IMAGE_COUNT), + [contents], + ); + + if (candidates.length === 0) return null; + + return ( +
+ {candidates.map((candidate) => ( + + ))} +
+ ); +}; diff --git a/packages/ui/src/components/chat/MarkdownRenderer.tsx b/packages/ui/src/components/chat/MarkdownRenderer.tsx index c294bd6f..00c0e80a 100644 --- a/packages/ui/src/components/chat/MarkdownRenderer.tsx +++ b/packages/ui/src/components/chat/MarkdownRenderer.tsx @@ -17,6 +17,10 @@ const SimpleMarkdownRendererLazy = lazyWithChunkRecovery(() => loadMarkdownRendererModule().then((m) => ({ default: m.SimpleMarkdownRenderer })) ); +const MarkdownImageGalleryLazy = lazyWithChunkRecovery(() => + import('./MarkdownImageGallery').then((m) => ({ default: m.MarkdownImageGallery })) +); + const fallback =
; const fallbackContentClassName = (variant: unknown): string => { @@ -48,3 +52,9 @@ export const SimpleMarkdownRenderer: React.FC ); + +export const MarkdownImageGallery: React.FC> = (props) => ( + + + +); diff --git a/packages/ui/src/components/chat/MarkdownRendererImpl.tsx b/packages/ui/src/components/chat/MarkdownRendererImpl.tsx index 0a8bef4c..a56f165b 100644 --- a/packages/ui/src/components/chat/MarkdownRendererImpl.tsx +++ b/packages/ui/src/components/chat/MarkdownRendererImpl.tsx @@ -19,7 +19,8 @@ import { isDesktopLocalOriginActive, isDesktopShell, isVSCodeRuntime } from '@/l import { isMobileSurfaceRuntime } from '@/lib/runtimeSurface'; import { ensureOutsideFileGrantForDesktop } from '@/lib/outsideFileGrants'; import { getDirectoryForFilePath, isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils'; -import { renderMarkdownBlocks, renderMarkdownSync } from './markdown/markdownCore'; +import { getMarkdownImageFilename, renderMarkdownBlocks, renderMarkdownSync } from './markdown/markdownCore'; +import { resolveMarkdownImageSource } from './markdown/markdownImageAssets'; import { ensureMarkdownShikiTheme } from './markdown/markdownTheme'; import { getMarkdownSyntaxVars } from './markdown/markdownSyntaxVars'; import { @@ -107,6 +108,59 @@ const useExternalLinkInteractions = ({ }, [containerRef, enabled]); }; +const useMarkdownImageLinkInteractions = ({ + containerRef, + directory, + enabled, + onShowPopup, +}: { + containerRef: React.RefObject; + directory: string; + enabled: boolean; + onShowPopup?: (content: ToolPopupContent) => void; +}) => { + React.useEffect(() => { + const container = containerRef.current; + if (!enabled || !container || !onShowPopup) return; + + const controller = new AbortController(); + const handleClick = (event: MouseEvent) => { + if (event.defaultPrevented || event.button !== 0) return; + const target = event.target; + if (!(target instanceof Element)) return; + + const link = target.closest('[data-openchamber-markdown-image-link="true"]'); + if (!link || !container.contains(link)) return; + + const source = link.getAttribute('data-openchamber-markdown-image-source') ?? ''; + const filename = link.getAttribute('data-openchamber-markdown-image-filename') + || getMarkdownImageFilename(source, ''); + if (!source || !filename) return; + + event.preventDefault(); + event.stopPropagation(); + void resolveMarkdownImageSource(source, directory, controller.signal) + .then((url) => { + if (controller.signal.aborted || !link.isConnected) return; + onShowPopup({ + open: true, + title: filename, + content: '', + metadata: { tool: 'markdown-image-preview', filename }, + image: { url, filename }, + }); + }) + .catch(() => undefined); + }; + + container.addEventListener('click', handleClick); + return () => { + controller.abort(); + container.removeEventListener('click', handleClick); + }; + }, [containerRef, directory, enabled, onShowPopup]); +}; + const DEFAULT_MERMAID_CONTROLS: MermaidControlOptions = { download: true, copy: true, @@ -140,6 +194,7 @@ interface MarkdownRendererProps { variant?: MarkdownVariant; onShowPopup?: (content: ToolPopupContent) => void; enableFileReferences?: boolean; + enableLocalImages?: boolean; } const FILE_LINK_SELECTOR = '[data-openchamber-file-link="true"]'; @@ -497,6 +552,10 @@ const useFileReferenceInteractions = ({ let linkedCount = 0; for (const candidate of Array.from(candidates)) { + if (candidate.matches('[data-openchamber-markdown-image-link="true"]')) { + clearFileLinkAttributes(candidate); + continue; + } const rawCandidate = extractPathCandidateFromElement(candidate); const resolved = getResolvedReference(rawCandidate, effectiveDirectory); clearFileLinkAttributes(candidate); @@ -831,6 +890,7 @@ const useMorphdomMarkdown = ({ text, streaming, cacheKey, + deferImages = false, syntaxVars, ctx, }: { @@ -838,6 +898,7 @@ const useMorphdomMarkdown = ({ text: string; streaming: boolean; cacheKey: string; + deferImages?: boolean; syntaxVars: Record; ctx: DecorateContext; }) => { @@ -876,7 +937,7 @@ const useMorphdomMarkdown = ({ // `display:contents` keeps margin-collapsing/spacing identical to a flat // HTML body — the wrapper exists only for per-block reconciliation. block.style.display = 'contents'; - block.innerHTML = renderMarkdownSync(text); + block.innerHTML = renderMarkdownSync(text, deferImages); // Decorate synchronously too: wrap code blocks in their framed card, // mark inline code, build table controls, etc. The async pass re-decorates // its own DOM before morphing, so without this the first paint shows bare @@ -888,7 +949,7 @@ const useMorphdomMarkdown = ({ refreshMermaidViewers(); } } - }, [containerRef, text, ctx, refreshMermaidViewers]); + }, [containerRef, text, deferImages, ctx, refreshMermaidViewers]); React.useEffect(() => () => { mermaidViewerRef.current?.cleanup(); @@ -901,7 +962,7 @@ const useMorphdomMarkdown = ({ const target = container.querySelector('[data-markdown-content]') ?? container; let active = true; - void renderMarkdownBlocks(text, streaming, cacheKey).then((blocks) => { + void renderMarkdownBlocks(text, streaming, cacheKey, deferImages).then((blocks) => { if (!active) return; const existing = Array.from(target.children) as HTMLElement[]; @@ -952,7 +1013,7 @@ const useMorphdomMarkdown = ({ return () => { active = false; }; - }, [containerRef, text, streaming, cacheKey, ctx, refreshMermaidViewers]); + }, [containerRef, text, streaming, cacheKey, deferImages, ctx, refreshMermaidViewers]); React.useEffect(() => { const container = containerRef.current; @@ -999,6 +1060,7 @@ const MarkdownRendererImpl: React.FC = ({ variant = 'assistant', onShowPopup, enableFileReferences = true, + enableLocalImages = false, }) => { streamPerfCount('ui.markdown_renderer.render'); if (isStreaming) streamPerfCount('ui.markdown_renderer.render.streaming'); @@ -1030,15 +1092,33 @@ const MarkdownRendererImpl: React.FC = ({ enabled: enableFileReferences && !isStreaming, }); useExternalLinkInteractions({ containerRef }); + useMarkdownImageLinkInteractions({ + containerRef, + directory: effectiveDirectory, + enabled: enableLocalImages && variant === 'assistant' && !isStreaming, + onShowPopup, + }); const syntaxVars = React.useMemo(() => getMarkdownSyntaxVars(currentTheme), [currentTheme]); const ctx = useDecorateContext(currentTheme, live, effectiveDirectory ? handlePreviewLoopback : undefined, DEFAULT_MERMAID_CONTROLS); const cacheKey = `markdown-${part?.id ? `part-${part.id}` : `message-${messageId}`}`; - useMorphdomMarkdown({ containerRef, text: content, streaming: live, cacheKey, syntaxVars, ctx }); + useMorphdomMarkdown({ + containerRef, + text: content, + streaming: live, + cacheKey, + deferImages: enableLocalImages && variant === 'assistant' && !isStreaming, + syntaxVars, + ctx, + }); const markdownContent = ( -
+
); @@ -1065,6 +1145,7 @@ export const MarkdownRenderer = React.memo(MarkdownRendererImpl, (prev, next) => && prev.messageId === next.messageId && prev.onShowPopup === next.onShowPopup && prev.enableFileReferences === next.enableFileReferences + && prev.enableLocalImages === next.enableLocalImages && prev.part?.id === next.part?.id; }); diff --git a/packages/ui/src/components/chat/markdown/markdownCore.test.ts b/packages/ui/src/components/chat/markdown/markdownCore.test.ts index 86db0bbd..84bd6eba 100644 --- a/packages/ui/src/components/chat/markdown/markdownCore.test.ts +++ b/packages/ui/src/components/chat/markdown/markdownCore.test.ts @@ -1,7 +1,21 @@ -import { describe, expect, test } from 'bun:test'; +import { describe, expect, mock, test } from 'bun:test'; + +mock.module('dompurify', () => ({ + default: { + isSupported: true, + addHook: () => undefined, + sanitize: (html: string) => html, + }, +})); +mock.module('./markdown-worker', () => ({ + highlightCodeInWorker: async () => null, +})); import { escapeRawMarkdownHtml, MARKDOWN_FORBIDDEN_TAGS } from './markdownSecurity'; +const { extractMarkdownImageCandidates, renderMarkdownSync } = await import('./markdownCore'); +const { resolveMarkdownImageSource } = await import('./markdownImageAssets'); + describe('markdown sanitization', () => { test('turns raw assistant HTML into inert visible text', () => { const payload = ''; @@ -16,3 +30,102 @@ describe('markdown sanitization', () => { expect(MARKDOWN_FORBIDDEN_TAGS).toContain('style'); }); }); + +describe('Markdown images', () => { + test('keeps local image links in text and emits inert image placeholders', () => { + const html = renderMarkdownSync([ + '[linked image](packages/vscode/extension.jpg)', + '![image syntax](packages/vscode/extension.jpg)', + ].join('\n\n'), true); + + expect(html).toContain('data-openchamber-markdown-image-link="true"'); + expect(html.match(/data-openchamber-markdown-image-source="packages\/vscode\/extension.jpg"/g)).toHaveLength(1); + expect(html).toContain('data-openchamber-markdown-image-placeholder="true"'); + expect(html).toContain('image syntax'); + expect(html).not.toContain('src="packages/vscode/extension.jpg"'); + expect(html).not.toContain('data-openchamber-markdown-image-state'); + expect(html.match(/ { + const html = renderMarkdownSync([ + '[remote link](https://example.test/image.png)', + '![remote image](https://example.test/image.png)', + ].join('\n\n'), true); + + expect(html).toContain(' { + const html = renderMarkdownSync([ + '![file](file:///workspace/image.png)', + '![unsafe](javascript:alert(1))', + ].join('\n\n'), true); + + expect(html).toContain('role="img"'); + expect(html).not.toContain('src="file:'); + expect(html).not.toContain('src="javascript:'); + }); + + test('collects a single ordered gallery across mixed Markdown and ignores code', () => { + const candidates = extractMarkdownImageCandidates([ + [ + 'Before [local link](screens/first%20view.png) and `![code](ignored.png)`.', + '', + '- ![duplicate](screens/first%20view.png)', + '- ![remote](https://example.test/second.webp?size=2)', + '', + '```md', + '![fenced](ignored-too.jpg)', + '```', + ].join('\n'), + 'After ![third](data:image/png;base64,AAAA).', + ]); + + expect(candidates).toEqual([ + { source: 'screens/first%20view.png', filename: 'first view.png' }, + { source: 'https://example.test/second.webp?size=2', filename: 'second.webp' }, + { source: 'data:image/png;base64,AAAA', filename: 'third' }, + ]); + }); + + test('limits one finalized message gallery to twelve unique candidates', () => { + const markdown = Array.from({ length: 14 }, (_, index) => `![image ${index}](screens/${index}.png)`).join('\n'); + + const candidates = extractMarkdownImageCandidates([markdown]); + + expect(candidates).toHaveLength(12); + expect(candidates.at(-1)?.source).toBe('screens/11.png'); + }); + + test('validates embedded image bytes against the declared MIME type', async () => { + const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; + const signal = new AbortController().signal; + + expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, '', signal)).toBe(`data:image/png;base64,${png}`); + await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, '', signal).then( + () => { throw new Error('Expected mismatched image data to fail'); }, + (error: unknown) => expect((error as Error).message).toBe('Unsupported image data'), + ); + }); + + test('does not resolve images after cancellation', async () => { + const controller = new AbortController(); + controller.abort(); + + await resolveMarkdownImageSource('https://example.test/image.png', '', controller.signal).then( + () => { throw new Error('Expected an aborted image load to fail'); }, + (error: unknown) => expect((error as Error).name).toBe('AbortError'), + ); + }); + + test('keeps the existing image renderer outside finalized assistant text', () => { + const html = renderMarkdownSync('![tool image](https://example.test/image.png)'); + + expect(html).toContain(' value.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); +const LOCAL_IMAGE_EXTENSION_RE = /\.(?:png|jpe?g|gif|webp)(?:[?#].*)?$/i; +const WINDOWS_ABSOLUTE_PATH_RE = /^[A-Za-z]:[\\/]/; +const URL_SCHEME_RE = /^[A-Za-z][A-Za-z\d+.-]*:/; + +export interface MarkdownImageCandidate { + source: string; + filename: string; +} + +export const MAX_MARKDOWN_IMAGE_COUNT = 12; + +const isLocalMarkdownImageSource = (source: string): boolean => { + if (/^\/\//.test(source) || !LOCAL_IMAGE_EXTENSION_RE.test(source)) return false; + return WINDOWS_ABSOLUTE_PATH_RE.test(source) + || /^file:\/\//i.test(source) + || !URL_SCHEME_RE.test(source); +}; + +const isSupportedMarkdownImageSource = (source: string): boolean => ( + /^(?:https?:)?\/\//i.test(source) + || /^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source) + || isLocalMarkdownImageSource(source) +); + +export const getMarkdownImageFilename = (source: string, fallback: string): string => { + if (/^data:/i.test(source)) return fallback.trim(); + + const path = source.split(/[?#]/, 1)[0]?.replace(/\\/g, '/') ?? ''; + const encodedName = path.split('/').filter(Boolean).at(-1) ?? ''; + if (!encodedName) return fallback.trim(); + try { + return decodeURIComponent(encodedName); + } catch { + return encodedName; + } +}; + +export const extractMarkdownImageCandidates = ( + markdownTexts: readonly string[], + limit = MAX_MARKDOWN_IMAGE_COUNT, +): MarkdownImageCandidate[] => { + if (limit <= 0) return []; + + const candidates: MarkdownImageCandidate[] = []; + const seen = new Set(); + + for (const markdown of markdownTexts) { + if (!markdown || candidates.length >= limit) continue; + const tokens = marked.lexer(markdown); + marked.walkTokens(tokens, (token) => { + if (candidates.length >= limit) return; + + if (token.type !== 'image' && token.type !== 'link') return; + if (token.type === 'link' && !isLocalMarkdownImageSource(token.href ?? '')) return; + + const source = token.href ?? ''; + if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return; + const fallback = typeof token.text === 'string' ? token.text : ''; + const filename = getMarkdownImageFilename(source, fallback); + if (!filename) return; + + seen.add(source); + candidates.push({ source, filename }); + }); + } + + return candidates; +}; + +const renderMarkdownImage = ({ + href, + title, + text, +}: { + href: string; + title?: string | null; + text: string; +}): string => { + const source = href ?? ''; + const alt = escapeAttr(text ?? ''); + const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; + const supported = isSupportedMarkdownImageSource(source); + if (!supported) { + return `${alt}`; + } + + return `${alt}`; +}; + // --------------------------------------------------------------------------- // Streaming block segmentation (port of OpenCode's markdown-stream) // --------------------------------------------------------------------------- @@ -162,7 +251,7 @@ const blockMathExtension = { }, }; -const parser = marked.use({ +const createParser = (deferImages: boolean) => new Marked().use({ gfm: true, breaks: false, extensions: [inlineMathExtension, blockMathExtension], @@ -175,6 +264,11 @@ const parser = marked.use({ }, link({ href, title, text }) { const target = href ?? ''; + if (deferImages && isLocalMarkdownImageSource(target)) { + const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; + const filename = getMarkdownImageFilename(target, ''); + return `${text}`; + } const agentName = parseAgentHref(target); if (agentName) { return `${text}`; @@ -186,9 +280,13 @@ const parser = marked.use({ const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; return `${text}`; }, + ...(deferImages ? { image: renderMarkdownImage } : {}), }, }); +const parser = createParser(false); +const imageParser = createParser(true); + // --------------------------------------------------------------------------- // Math (KaTeX) — post-process the parsed HTML, skipping code/pre/kbd content // --------------------------------------------------------------------------- @@ -341,8 +439,8 @@ const touch = (key: string, entry: { hash: string; html: string }): void => { if (oldest) htmlCache.delete(oldest); }; -const parseBlock = async (block: MarkdownBlock): Promise => { - const parsed = await Promise.resolve(parser.parse(block.src)); +const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise => { + const parsed = await Promise.resolve((deferImages ? imageParser : parser).parse(block.src)); const withMath = renderMathExpressions(parsed); const highlighted = block.highlight ? await highlightCodeBlocks(withMath) : withMath; return sanitize(highlighted); @@ -357,9 +455,9 @@ const parseBlock = async (block: MarkdownBlock): Promise => { * is synchronous (marked is not configured `async`), so this never blocks on a * worker round-trip. */ -export const renderMarkdownSync = (text: string): string => { +export const renderMarkdownSync = (text: string, deferImages = false): string => { if (!text) return ''; - const parsed = parser.parse(text) as string; + const parsed = (deferImages ? imageParser : parser).parse(text) as string; const withMath = renderMathExpressions(parsed); return sanitize(withMath); }; @@ -382,6 +480,7 @@ export const renderMarkdownBlocks = async ( text: string, streaming: boolean, cacheKey: string, + deferImages = false, ): Promise => { if (!text) return []; @@ -389,14 +488,14 @@ export const renderMarkdownBlocks = async ( return Promise.all( blocks.map(async (block, index) => { const contentHash = hash(block.raw); - const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}`; - const key = `${cacheKey}:${index}:${block.mode}`; + const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${deferImages ? 1 : 0}`; + const key = `${cacheKey}:${index}:${block.mode}:${deferImages ? 1 : 0}`; const cached = htmlCache.get(key); if (cached && cached.hash === contentHash) { touch(key, cached); return { id, html: cached.html }; } - const html = await parseBlock(block); + const html = await parseBlock(block, deferImages); touch(key, { hash: contentHash, html }); return { id, html }; }), diff --git a/packages/ui/src/components/chat/markdown/markdownImageAssets.ts b/packages/ui/src/components/chat/markdown/markdownImageAssets.ts new file mode 100644 index 00000000..e61fea18 --- /dev/null +++ b/packages/ui/src/components/chat/markdown/markdownImageAssets.ts @@ -0,0 +1,132 @@ +import { isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils'; +import { runtimeFetch } from '@/lib/runtime-fetch'; + +const MAX_MARKDOWN_IMAGE_BYTES = 10 * 1024 * 1024; +const SUPPORTED_IMAGE_MIME_TYPES = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]); + +const parseLocalImagePath = (source: string): string => { + let value = source; + if (/^file:\/\//i.test(value)) { + try { + const fileUrl = new URL(value); + if (fileUrl.protocol !== 'file:') return ''; + value = fileUrl.host && fileUrl.host !== 'localhost' + ? `//${fileUrl.host}${fileUrl.pathname}` + : fileUrl.pathname; + if (/^\/[A-Za-z]:\//.test(value)) value = value.slice(1); + } catch { + return ''; + } + } + + const path = value.split(/[?#]/, 1)[0] ?? ''; + try { + return decodeURIComponent(path); + } catch { + return path; + } +}; + +const blobToDataUrl = (blob: Blob): Promise => new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => typeof reader.result === 'string' + ? resolve(reader.result) + : reject(new Error('Unable to encode image')); + reader.onerror = () => reject(reader.error ?? new Error('Unable to encode image')); + reader.readAsDataURL(blob); +}); + +const hasImageSignature = async (blob: Blob, mimeType: string): Promise => { + const bytes = new Uint8Array(await blob.slice(0, 12).arrayBuffer()); + const ascii = (start: number, end: number) => String.fromCharCode(...bytes.slice(start, end)); + if (mimeType === 'image/png') { + return bytes[0] === 0x89 && ascii(1, 4) === 'PNG' + && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a; + } + if (mimeType === 'image/jpeg') { + return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff; + } + if (mimeType === 'image/gif') { + const gif = ascii(0, 6); + return gif === 'GIF87a' || gif === 'GIF89a'; + } + return mimeType === 'image/webp' && ascii(0, 4) === 'RIFF' && ascii(8, 12) === 'WEBP'; +}; + +const validateImageBlob = async (blob: Blob, mimeType: string): Promise => { + if (!SUPPORTED_IMAGE_MIME_TYPES.has(mimeType)) throw new Error('Unsupported image type'); + if (blob.size > MAX_MARKDOWN_IMAGE_BYTES) throw new Error('Image is too large'); + if (!await hasImageSignature(blob, mimeType)) throw new Error('Unsupported image data'); +}; + +const validateDataImage = async (source: string): Promise => { + const match = /^data:(image\/(?:png|jpeg|gif|webp));base64,([\s\S]*)$/i.exec(source); + if (!match?.[1] || match[2] === undefined) throw new Error('Invalid image data URL'); + const encoded = match[2]; + if (encoded.length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) { + throw new Error('Image is too large'); + } + + let binary: string; + try { + binary = atob(encoded); + } catch { + throw new Error('Invalid image data URL'); + } + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + await validateImageBlob(new Blob([bytes]), match[1].toLowerCase()); +}; + +export const resolveMarkdownImageSource = async ( + source: string, + directory: string, + signal: AbortSignal, +): Promise => { + if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError'); + if (/^(?:https?:)?\/\//i.test(source)) return source; + + if (/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)) { + await validateDataImage(source); + if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError'); + return source; + } + + const localPath = parseLocalImagePath(source); + const absolutePath = toAbsoluteFilePath(directory, localPath); + if (!directory || !localPath || !isFilePathWithinDirectory(absolutePath, directory)) { + throw new Error('Image path is outside the active workspace'); + } + + const statResponse = await runtimeFetch('/api/fs/stat', { + query: { path: absolutePath, directory, optional: 'true' }, + signal, + }); + if (!statResponse.ok) throw new Error(`Unable to inspect image (${statResponse.status})`); + const stat = await statResponse.json() as { isFile?: boolean; size?: number }; + if (!stat.isFile) throw new Error('Image path is not a file'); + if (typeof stat.size === 'number' && stat.size > MAX_MARKDOWN_IMAGE_BYTES) { + throw new Error('Image is too large'); + } + + const response = await runtimeFetch('/api/fs/raw', { + query: { path: absolutePath, directory }, + signal, + }); + if (!response.ok) throw new Error(`Unable to load image (${response.status})`); + + const mimeType = (response.headers.get('content-type') ?? '').split(';', 1)[0]?.toLowerCase() ?? ''; + const contentLength = Number(response.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > MAX_MARKDOWN_IMAGE_BYTES) { + throw new Error('Image is too large'); + } + + const blob = await response.blob(); + await validateImageBlob(blob, mimeType); + return blobToDataUrl(blob); +}; diff --git a/packages/ui/src/components/chat/message/MessageBody.tsx b/packages/ui/src/components/chat/message/MessageBody.tsx index d2e04411..a697c165 100644 --- a/packages/ui/src/components/chat/message/MessageBody.tsx +++ b/packages/ui/src/components/chat/message/MessageBody.tsx @@ -21,7 +21,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip import { ArrowsMerge } from '@/components/icons/ArrowsMerge'; import type { ContentChangeReason } from '@/hooks/useChatAutoFollow'; -import { SimpleMarkdownRenderer } from '../MarkdownRenderer'; +import { MarkdownImageGallery, SimpleMarkdownRenderer } from '../MarkdownRenderer'; import { useSessionUIStore } from '@/sync/session-ui-store'; import { useUIStore } from '@/stores/useUIStore'; import { flattenAssistantTextParts, suggestPlanTitleFromText } from '@/lib/messages/messageText'; @@ -1211,6 +1211,11 @@ const AssistantMessageBody = React.memo(({ const assistantTextParts = React.useMemo(() => { return visibleParts.filter((part) => part.type === 'text'); }, [visibleParts]); + const finalizedAssistantMarkdownContents = React.useMemo(() => ( + isMessageCompleted + ? assistantTextParts.map(extractTextContent).filter((text) => text.trim().length > 0) + : [] + ), [assistantTextParts, isMessageCompleted]); const assistantPlanText = React.useMemo(() => flattenAssistantTextParts(assistantTextParts), [assistantTextParts]); const suggestedPlanTitle = React.useMemo(() => suggestPlanTitleFromText(assistantPlanText), [assistantPlanText]); @@ -1863,6 +1868,7 @@ const AssistantMessageBody = React.memo(({ chatRenderMode={chatRenderMode} onContentChange={onContentChange} onShowPopup={onShowPopup} + enableMarkdownImages={isMessageCompleted} />
); @@ -2017,6 +2023,7 @@ const AssistantMessageBody = React.memo(({ collapsedPreviewCount, expandedTools, isMobile, + isMessageCompleted, isActivityOwnerMessage, isSortedRenderMode, lastRenderableTextPartIndex, @@ -2228,6 +2235,10 @@ const AssistantMessageBody = React.memo(({ )}
+ {shouldRenderStandaloneActionsAfterContent && (
diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts index 08584eb8..a0d7f916 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test'; +import { getImagePreviewBounds, getImagePreviewDialogLayout } from './imagePreviewSizing'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; describe('getMermaidDataUrlSourcePromise', () => { @@ -29,3 +30,43 @@ describe('Mermaid load request ids', () => { expect(isCurrentMermaidLoadRequest(secondRequest, secondRequest)).toBe(true); }); }); + +describe('Markdown image preview bounds', () => { + test('uses sixty percent of the viewport width with vertical containment', () => { + expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, true)).toEqual({ + maxWidth: 720, + maxHeight: 640, + }); + }); + + test('preserves existing attachment preview bounds', () => { + expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, false)).toEqual({ + maxWidth: 900, + maxHeight: 600, + }); + }); + + test('keeps a readable modal width for narrow portrait images', () => { + expect(getImagePreviewDialogLayout( + { width: 29, height: 576 }, + { width: 1280, height: 720 }, + false, + )).toEqual({ + dialogWidth: 320, + imageWidth: 29, + imageHeight: 576, + }); + }); + + test('fits image content inside the mobile dialog chrome without cropping', () => { + expect(getImagePreviewDialogLayout( + { width: 275, height: 500 }, + { width: 320, height: 700 }, + true, + )).toEqual({ + dialogWidth: 304, + imageWidth: 270, + imageHeight: 491, + }); + }); +}); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx index 15298662..ce01e48b 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Dialog, DialogContent } from '@/components/ui/dialog'; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; import { File as PierreFile, PatchDiff } from '@pierre/diffs/react'; import { WorkerHighlightedCode } from '@/components/code/WorkerHighlightedCode'; import { createPortal } from 'react-dom'; @@ -29,6 +29,11 @@ import { Icon } from "@/components/icon/Icon"; import { useI18n, type I18nKey, type I18nParams } from '@/lib/i18n'; import { runtimeFetch } from '@/lib/runtime-fetch'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, isMermaidLoadFailure, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; +import { + getImagePreviewBounds, + getImagePreviewDialogLayout, + type ImagePreviewViewport, +} from './imagePreviewSizing'; interface ToolOutputDialogProps { popup: ToolPopupContent; @@ -158,7 +163,7 @@ const usePierreThemeConfig = (): PierreThemeConfig => { }; }; -type ViewportSize = { width: number; height: number }; +type ViewportSize = ImagePreviewViewport; const getWindowViewport = (): ViewportSize => ({ width: typeof window !== 'undefined' ? window.innerWidth : 0, @@ -331,8 +336,11 @@ const ImagePreviewDialog: React.FC<{ }, [popup.image]); const [currentIndex, setCurrentIndex] = React.useState(0); - const [imageNaturalSize, setImageNaturalSize] = React.useState<{ width: number; height: number } | null>(null); - const { isRendered, isVisible, isTransitioning } = usePreviewOverlayState(popup.open); + const [loadedImageSize, setLoadedImageSize] = React.useState<{ + url: string; + width: number; + height: number; + } | null>(null); const viewport = usePreviewViewport(popup.open); React.useEffect(() => { @@ -352,9 +360,10 @@ const ImagePreviewDialog: React.FC<{ setCurrentIndex(matchingIndex >= 0 ? matchingIndex : 0); }, [gallery, popup.image?.index, popup.image?.url, popup.open]); - const currentImage = gallery[currentIndex] ?? gallery[0] ?? popup.image; + const currentImage = gallery[currentIndex] ?? gallery[0]; const imageTitle = currentImage?.filename || popup.title || 'Image preview'; const hasMultipleImages = gallery.length > 1; + const markdownImage = popup.metadata?.tool === 'markdown-image-preview'; const showPrevious = React.useCallback(() => { if (gallery.length <= 1) return; @@ -372,11 +381,6 @@ const ImagePreviewDialog: React.FC<{ } const onKeyDown = (event: KeyboardEvent) => { - if (event.key === 'Escape') { - onOpenChange(false); - return; - } - if (event.key === 'ArrowLeft' && hasMultipleImages) { event.preventDefault(); showPrevious(); @@ -393,126 +397,108 @@ const ImagePreviewDialog: React.FC<{ return () => { window.removeEventListener('keydown', onKeyDown); }; - }, [hasMultipleImages, onOpenChange, popup.open, showNext, showPrevious]); + }, [hasMultipleImages, popup.open, showNext, showPrevious]); - React.useEffect(() => { - setImageNaturalSize(null); - }, [currentImage?.url]); - - const imageDisplaySize = React.useMemo(() => { - const maxWidth = Math.max(160, viewport.width * (isMobile ? 0.86 : 0.75)); - const maxHeight = Math.max(160, viewport.height * (isMobile ? 0.72 : 0.75)); - - if (!imageNaturalSize) { - return { - width: Math.round(maxWidth), - height: Math.round(maxHeight), - }; - } + const imageNaturalSize = loadedImageSize?.url === currentImage?.url + ? loadedImageSize + : null; + const { maxWidth, maxHeight } = getImagePreviewBounds(viewport, isMobile, markdownImage); + let imageDisplaySize = { + width: Math.round(maxWidth), + height: Math.round(maxHeight), + }; + if (imageNaturalSize) { const widthScale = maxWidth / imageNaturalSize.width; const heightScale = maxHeight / imageNaturalSize.height; const scale = Math.min(widthScale, heightScale); - - return { + imageDisplaySize = { width: Math.max(1, Math.round(imageNaturalSize.width * scale)), height: Math.max(1, Math.round(imageNaturalSize.height * scale)), }; - }, [imageNaturalSize, isMobile, viewport.height, viewport.width]); + } - if (!isRendered || !currentImage || typeof document === 'undefined') { + const handleImageLoad = React.useCallback((event: React.SyntheticEvent) => { + const element = event.currentTarget; + const width = element.naturalWidth; + const height = element.naturalHeight; + if (width <= 0 || height <= 0) return; + + const url = element.getAttribute('src') ?? ''; + setLoadedImageSize((previous) => { + if (previous && previous.url === url && previous.width === width && previous.height === height) { + return previous; + } + return { url, width, height }; + }); + }, []); + + if (!currentImage) { return null; } - const content = ( -
- ); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts index 17d62a75..a0d7f916 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts @@ -1,13 +1,6 @@ import { describe, expect, test } from 'bun:test'; -import { - clampImagePreviewTransform, - getContainedImagePreviewSize, - getImagePreviewBounds, - getImagePreviewDialogLayout, - getImagePreviewGestureTransform, - getLocalImagePreviewPoints, -} from './imagePreviewSizing'; +import { getImagePreviewBounds, getImagePreviewDialogLayout } from './imagePreviewSizing'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; describe('getMermaidDataUrlSourcePromise', () => { @@ -77,52 +70,3 @@ describe('Markdown image preview bounds', () => { }); }); }); - -describe('Mobile image preview gestures', () => { - test('zooms around the midpoint of a two-finger gesture', () => { - expect(getImagePreviewGestureTransform( - { scale: 1, x: 0, y: 0 }, - [{ x: 100, y: 200 }, { x: 200, y: 200 }], - [{ x: 50, y: 200 }, { x: 250, y: 200 }], - { width: 300, height: 500 }, - { width: 300, height: 500 }, - )).toEqual({ scale: 2, x: 0, y: 50 }); - }); - - test('supports panning after zoom and clamps the image to the viewport', () => { - expect(getImagePreviewGestureTransform( - { scale: 2, x: 0, y: 0 }, - [{ x: 100, y: 100 }], - [{ x: 400, y: -400 }], - { width: 300, height: 500 }, - { width: 300, height: 500 }, - )).toEqual({ scale: 2, x: 150, y: -250 }); - }); - - test('limits pinch zoom to four times', () => { - expect(clampImagePreviewTransform( - { scale: 8, x: 1000, y: -1000 }, - { width: 300, height: 500 }, - { width: 300, height: 500 }, - )).toEqual({ scale: 4, x: 450, y: -750 }); - }); - - test('converts page coordinates into the image viewport coordinate system', () => { - expect(getLocalImagePreviewPoints( - [{ x: 129, y: 257 }, { x: 229, y: 257 }], - { x: 29, y: 57 }, - )).toEqual([{ x: 100, y: 200 }, { x: 200, y: 200 }]); - }); - - test('clamps letterboxed wide images against their visible content', () => { - const content = getContainedImagePreviewSize( - { width: 2908, height: 1686 }, - { width: 332, height: 758 }, - ); - expect(clampImagePreviewTransform( - { scale: 2.5, x: 1000, y: 1000 }, - { width: 332, height: 758 }, - content, - )).toEqual({ scale: 2.5, x: 249, y: 0 }); - }); -}); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx index d6a1ad30..ce01e48b 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx @@ -30,14 +30,8 @@ import { useI18n, type I18nKey, type I18nParams } from '@/lib/i18n'; import { runtimeFetch } from '@/lib/runtime-fetch'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, isMermaidLoadFailure, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; import { - clampImagePreviewTransform, - getContainedImagePreviewSize, - getImagePreviewGestureTransform, getImagePreviewBounds, getImagePreviewDialogLayout, - getLocalImagePreviewPoints, - IDENTITY_IMAGE_PREVIEW_TRANSFORM, - type ImagePreviewPoint, type ImagePreviewViewport, } from './imagePreviewSizing'; @@ -370,76 +364,6 @@ const ImagePreviewDialog: React.FC<{ const imageTitle = currentImage?.filename || popup.title || 'Image preview'; const hasMultipleImages = gallery.length > 1; const markdownImage = popup.metadata?.tool === 'markdown-image-preview'; - const mobileMarkdownViewer = isMobile && markdownImage; - const imageViewportRef = React.useRef(null); - const activePointersRef = React.useRef(new Map()); - const previousPointsRef = React.useRef([]); - const [imageTransform, setImageTransform] = React.useState(IDENTITY_IMAGE_PREVIEW_TRANSFORM); - - React.useEffect(() => { - activePointersRef.current.clear(); - previousPointsRef.current = []; - setImageTransform(IDENTITY_IMAGE_PREVIEW_TRANSFORM); - }, [currentImage?.url, popup.open]); - - const handleImagePointerDown = React.useCallback((event: React.PointerEvent) => { - if (!mobileMarkdownViewer || event.pointerType !== 'touch') return; - event.preventDefault(); - event.currentTarget.setPointerCapture(event.pointerId); - activePointersRef.current.set(event.pointerId, { x: event.clientX, y: event.clientY }); - previousPointsRef.current = Array.from(activePointersRef.current.values()); - }, [mobileMarkdownViewer]); - - const handleImagePointerMove = React.useCallback((event: React.PointerEvent) => { - if (!mobileMarkdownViewer || !activePointersRef.current.has(event.pointerId)) return; - event.preventDefault(); - activePointersRef.current.set(event.pointerId, { x: event.clientX, y: event.clientY }); - const currentPoints = Array.from(activePointersRef.current.values()); - const previousPoints = previousPointsRef.current; - const imageViewport = imageViewportRef.current; - if (imageViewport) { - const bounds = imageViewport.getBoundingClientRect(); - const viewportSize = { width: bounds.width, height: bounds.height }; - const image = imageViewport.querySelector('img'); - const contentSize = getContainedImagePreviewSize({ - width: image?.naturalWidth || bounds.width, - height: image?.naturalHeight || bounds.height, - }, viewportSize); - setImageTransform((current) => getImagePreviewGestureTransform( - current, - getLocalImagePreviewPoints(previousPoints, { x: bounds.left, y: bounds.top }), - getLocalImagePreviewPoints(currentPoints, { x: bounds.left, y: bounds.top }), - viewportSize, - contentSize, - )); - } - previousPointsRef.current = currentPoints; - }, [mobileMarkdownViewer]); - - const handleImagePointerEnd = React.useCallback((event: React.PointerEvent) => { - if (!mobileMarkdownViewer) return; - activePointersRef.current.delete(event.pointerId); - previousPointsRef.current = Array.from(activePointersRef.current.values()); - if (event.currentTarget.hasPointerCapture(event.pointerId)) { - event.currentTarget.releasePointerCapture(event.pointerId); - } - }, [mobileMarkdownViewer]); - - const adjustImageScale = React.useCallback((delta: number) => { - const imageViewport = imageViewportRef.current; - if (!imageViewport) return; - const bounds = imageViewport.getBoundingClientRect(); - const viewportSize = { width: bounds.width, height: bounds.height }; - const image = imageViewport.querySelector('img'); - const contentSize = getContainedImagePreviewSize({ - width: image?.naturalWidth || bounds.width, - height: image?.naturalHeight || bounds.height, - }, viewportSize); - setImageTransform((current) => clampImagePreviewTransform({ - ...current, - scale: current.scale + delta, - }, viewportSize, contentSize)); - }, []); const showPrevious = React.useCallback(() => { if (gallery.length <= 1) return; @@ -519,17 +443,10 @@ const ImagePreviewDialog: React.FC<{ button]:right-3 [&>button]:top-3', )} - style={mobileMarkdownViewer ? { - width: 'calc(100vw - 2rem)', - maxWidth: 'none', - maxHeight: 'none', - } : { + style={{ width: `${dialogLayout.dialogWidth}px`, maxWidth: isMobile ? 'calc(100vw - 1rem)' : 'calc(100vw - 2rem)', maxHeight: isMobile ? 'calc(100vh - 1rem)' : 'calc(100vh - 2rem)', @@ -546,19 +463,8 @@ const ImagePreviewDialog: React.FC<{
{hasMultipleImages ? ( <> @@ -584,51 +490,12 @@ const ImagePreviewDialog: React.FC<{ src={currentImage.url} alt={imageTitle} className="block h-full w-full object-contain" - style={mobileMarkdownViewer ? { - transform: `translate3d(${imageTransform.x}px, ${imageTransform.y}px, 0) scale(${imageTransform.scale})`, - transformOrigin: 'center', - } : undefined} loading="lazy" onLoad={handleImageLoad} data-openchamber-markdown-image-preview={markdownImage ? 'true' : undefined} - data-openchamber-image-preview-scale={mobileMarkdownViewer ? imageTransform.scale : undefined} /> - {mobileMarkdownViewer ? ( -
event.stopPropagation()} - > - - - -
- ) : null}
- {!mobileMarkdownViewer ? ); diff --git a/packages/ui/src/components/chat/message/imagePreviewSizing.ts b/packages/ui/src/components/chat/message/imagePreviewSizing.ts index 846ce09b..9ef842ee 100644 --- a/packages/ui/src/components/chat/message/imagePreviewSizing.ts +++ b/packages/ui/src/components/chat/message/imagePreviewSizing.ts @@ -1,10 +1,5 @@ export type ImagePreviewViewport = { width: number; height: number }; type ImagePreviewSize = { width: number; height: number }; -export type ImagePreviewPoint = { x: number; y: number }; -type ImagePreviewTransform = { scale: number; x: number; y: number }; - -export const IDENTITY_IMAGE_PREVIEW_TRANSFORM: ImagePreviewTransform = { scale: 1, x: 0, y: 0 }; -const MAX_IMAGE_PREVIEW_SCALE = 4; export const getImagePreviewBounds = ( viewport: ImagePreviewViewport, @@ -39,88 +34,3 @@ export const getImagePreviewDialogLayout = ( imageHeight: Math.max(1, Math.round(image.height * scale)), }; }; - -const midpoint = (first: ImagePreviewPoint, second: ImagePreviewPoint): ImagePreviewPoint => ({ - x: (first.x + second.x) / 2, - y: (first.y + second.y) / 2, -}); - -const distance = (first: ImagePreviewPoint, second: ImagePreviewPoint): number => ( - Math.hypot(second.x - first.x, second.y - first.y) -); - -export const getContainedImagePreviewSize = ( - image: ImagePreviewSize, - viewport: ImagePreviewViewport, -): ImagePreviewSize => { - const scale = Math.min( - viewport.width / Math.max(1, image.width), - viewport.height / Math.max(1, image.height), - ); - return { - width: image.width * scale, - height: image.height * scale, - }; -}; - -export const getLocalImagePreviewPoints = ( - points: ImagePreviewPoint[], - origin: ImagePreviewPoint, -): ImagePreviewPoint[] => points.map((point) => ({ - x: point.x - origin.x, - y: point.y - origin.y, -})); - -export const clampImagePreviewTransform = ( - transform: ImagePreviewTransform, - viewport: ImagePreviewViewport, - content: ImagePreviewSize, -): ImagePreviewTransform => { - const scale = Math.min(MAX_IMAGE_PREVIEW_SCALE, Math.max(1, transform.scale)); - const maxX = Math.max(0, (content.width * scale - viewport.width) / 2); - const maxY = Math.max(0, (content.height * scale - viewport.height) / 2); - - return { - scale, - x: Math.min(maxX, Math.max(-maxX, scale === 1 ? 0 : transform.x)), - y: Math.min(maxY, Math.max(-maxY, scale === 1 ? 0 : transform.y)), - }; -}; - -export const getImagePreviewGestureTransform = ( - transform: ImagePreviewTransform, - previousPoints: ImagePreviewPoint[], - currentPoints: ImagePreviewPoint[], - viewport: ImagePreviewViewport, - content: ImagePreviewSize, -): ImagePreviewTransform => { - if (previousPoints.length >= 2 && currentPoints.length >= 2) { - const previousDistance = distance(previousPoints[0], previousPoints[1]); - if (previousDistance <= 0) return transform; - - const previousMidpoint = midpoint(previousPoints[0], previousPoints[1]); - const currentMidpoint = midpoint(currentPoints[0], currentPoints[1]); - const scale = Math.min( - MAX_IMAGE_PREVIEW_SCALE, - Math.max(1, transform.scale * distance(currentPoints[0], currentPoints[1]) / previousDistance), - ); - const ratio = scale / transform.scale; - const center = { x: viewport.width / 2, y: viewport.height / 2 }; - - return clampImagePreviewTransform({ - scale, - x: currentMidpoint.x - center.x - (previousMidpoint.x - center.x - transform.x) * ratio, - y: currentMidpoint.y - center.y - (previousMidpoint.y - center.y - transform.y) * ratio, - }, viewport, content); - } - - if (previousPoints.length === 1 && currentPoints.length === 1 && transform.scale > 1) { - return clampImagePreviewTransform({ - ...transform, - x: transform.x + currentPoints[0].x - previousPoints[0].x, - y: transform.y + currentPoints[0].y - previousPoints[0].y, - }, viewport, content); - } - - return transform; -}; diff --git a/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md b/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md index 99c68046..f709d590 100644 --- a/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md +++ b/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md @@ -68,9 +68,9 @@ Use this doc when you ask an agent to change tool/header/description behavior. completed assistant message hydrates at most 12 unique image candidates, including persisted text parts that omit their optional part-level end time. The image modal reserves readable title width even for narrow portrait media. - Markdown image previews use the 60vw contained dialog on desktop and a - full-height mobile dialog with 1x-4x pinch zoom, zoom controls, and panning - when zoomed. + This gallery and modal-preview enhancement is desktop-only. Mobile keeps the + standard Markdown rendering path until a dedicated, non-modal full-screen + image viewer can own safe areas, orientation, and touch gestures coherently. - `read` and `skill` are **static navigation tools** and render via `StaticToolRow`. - Every other tool, including search/fetch, OpenCode built-ins, custom tools, plugins, and MCP tools, is **expandable** and renders through `ToolPart`. - The managed `openchamber` plugin tool uses the expandable path and hides its broad protocol input. The plugin supplies the selected action's human description as the native tool title; the UI renders that metadata without owning an action map. The full versioned result envelope renders through the same neutral JSON summary/tree/raw views as other tools, without a tool-specific output card. From 5d29ef15d31a38b2f9236741ebac2ecb51112ca7 Mon Sep 17 00:00:00 2001 From: ChangeHow Date: Thu, 13 Aug 2026 16:18:58 +0800 Subject: [PATCH 4/4] fix: reuse existing image preview for galleries --- .../components/chat/message/MessageBody.tsx | 6 +- .../chat/message/ToolOutputDialog.test.ts | 41 ---- .../chat/message/ToolOutputDialog.tsx | 198 ++++++++++-------- .../chat/message/imagePreviewSizing.ts | 36 ---- .../chat/message/parts/DOCUMENTATION.md | 11 +- 5 files changed, 114 insertions(+), 178 deletions(-) delete mode 100644 packages/ui/src/components/chat/message/imagePreviewSizing.ts diff --git a/packages/ui/src/components/chat/message/MessageBody.tsx b/packages/ui/src/components/chat/message/MessageBody.tsx index 8963e5fb..a697c165 100644 --- a/packages/ui/src/components/chat/message/MessageBody.tsx +++ b/packages/ui/src/components/chat/message/MessageBody.tsx @@ -1212,10 +1212,10 @@ const AssistantMessageBody = React.memo(({ return visibleParts.filter((part) => part.type === 'text'); }, [visibleParts]); const finalizedAssistantMarkdownContents = React.useMemo(() => ( - isMessageCompleted && !isMobile + isMessageCompleted ? assistantTextParts.map(extractTextContent).filter((text) => text.trim().length > 0) : [] - ), [assistantTextParts, isMessageCompleted, isMobile]); + ), [assistantTextParts, isMessageCompleted]); const assistantPlanText = React.useMemo(() => flattenAssistantTextParts(assistantTextParts), [assistantTextParts]); const suggestedPlanTitle = React.useMemo(() => suggestPlanTitleFromText(assistantPlanText), [assistantPlanText]); @@ -1868,7 +1868,7 @@ const AssistantMessageBody = React.memo(({ chatRenderMode={chatRenderMode} onContentChange={onContentChange} onShowPopup={onShowPopup} - enableMarkdownImages={isMessageCompleted && !isMobile} + enableMarkdownImages={isMessageCompleted} />
); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts index a0d7f916..08584eb8 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts @@ -1,6 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { getImagePreviewBounds, getImagePreviewDialogLayout } from './imagePreviewSizing'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; describe('getMermaidDataUrlSourcePromise', () => { @@ -30,43 +29,3 @@ describe('Mermaid load request ids', () => { expect(isCurrentMermaidLoadRequest(secondRequest, secondRequest)).toBe(true); }); }); - -describe('Markdown image preview bounds', () => { - test('uses sixty percent of the viewport width with vertical containment', () => { - expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, true)).toEqual({ - maxWidth: 720, - maxHeight: 640, - }); - }); - - test('preserves existing attachment preview bounds', () => { - expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, false)).toEqual({ - maxWidth: 900, - maxHeight: 600, - }); - }); - - test('keeps a readable modal width for narrow portrait images', () => { - expect(getImagePreviewDialogLayout( - { width: 29, height: 576 }, - { width: 1280, height: 720 }, - false, - )).toEqual({ - dialogWidth: 320, - imageWidth: 29, - imageHeight: 576, - }); - }); - - test('fits image content inside the mobile dialog chrome without cropping', () => { - expect(getImagePreviewDialogLayout( - { width: 275, height: 500 }, - { width: 320, height: 700 }, - true, - )).toEqual({ - dialogWidth: 304, - imageWidth: 270, - imageHeight: 491, - }); - }); -}); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx index ce01e48b..15298662 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; +import { Dialog, DialogContent } from '@/components/ui/dialog'; import { File as PierreFile, PatchDiff } from '@pierre/diffs/react'; import { WorkerHighlightedCode } from '@/components/code/WorkerHighlightedCode'; import { createPortal } from 'react-dom'; @@ -29,11 +29,6 @@ import { Icon } from "@/components/icon/Icon"; import { useI18n, type I18nKey, type I18nParams } from '@/lib/i18n'; import { runtimeFetch } from '@/lib/runtime-fetch'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, isMermaidLoadFailure, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; -import { - getImagePreviewBounds, - getImagePreviewDialogLayout, - type ImagePreviewViewport, -} from './imagePreviewSizing'; interface ToolOutputDialogProps { popup: ToolPopupContent; @@ -163,7 +158,7 @@ const usePierreThemeConfig = (): PierreThemeConfig => { }; }; -type ViewportSize = ImagePreviewViewport; +type ViewportSize = { width: number; height: number }; const getWindowViewport = (): ViewportSize => ({ width: typeof window !== 'undefined' ? window.innerWidth : 0, @@ -336,11 +331,8 @@ const ImagePreviewDialog: React.FC<{ }, [popup.image]); const [currentIndex, setCurrentIndex] = React.useState(0); - const [loadedImageSize, setLoadedImageSize] = React.useState<{ - url: string; - width: number; - height: number; - } | null>(null); + const [imageNaturalSize, setImageNaturalSize] = React.useState<{ width: number; height: number } | null>(null); + const { isRendered, isVisible, isTransitioning } = usePreviewOverlayState(popup.open); const viewport = usePreviewViewport(popup.open); React.useEffect(() => { @@ -360,10 +352,9 @@ const ImagePreviewDialog: React.FC<{ setCurrentIndex(matchingIndex >= 0 ? matchingIndex : 0); }, [gallery, popup.image?.index, popup.image?.url, popup.open]); - const currentImage = gallery[currentIndex] ?? gallery[0]; + const currentImage = gallery[currentIndex] ?? gallery[0] ?? popup.image; const imageTitle = currentImage?.filename || popup.title || 'Image preview'; const hasMultipleImages = gallery.length > 1; - const markdownImage = popup.metadata?.tool === 'markdown-image-preview'; const showPrevious = React.useCallback(() => { if (gallery.length <= 1) return; @@ -381,6 +372,11 @@ const ImagePreviewDialog: React.FC<{ } const onKeyDown = (event: KeyboardEvent) => { + if (event.key === 'Escape') { + onOpenChange(false); + return; + } + if (event.key === 'ArrowLeft' && hasMultipleImages) { event.preventDefault(); showPrevious(); @@ -397,108 +393,126 @@ const ImagePreviewDialog: React.FC<{ return () => { window.removeEventListener('keydown', onKeyDown); }; - }, [hasMultipleImages, popup.open, showNext, showPrevious]); + }, [hasMultipleImages, onOpenChange, popup.open, showNext, showPrevious]); - const imageNaturalSize = loadedImageSize?.url === currentImage?.url - ? loadedImageSize - : null; + React.useEffect(() => { + setImageNaturalSize(null); + }, [currentImage?.url]); + + const imageDisplaySize = React.useMemo(() => { + const maxWidth = Math.max(160, viewport.width * (isMobile ? 0.86 : 0.75)); + const maxHeight = Math.max(160, viewport.height * (isMobile ? 0.72 : 0.75)); + + if (!imageNaturalSize) { + return { + width: Math.round(maxWidth), + height: Math.round(maxHeight), + }; + } - const { maxWidth, maxHeight } = getImagePreviewBounds(viewport, isMobile, markdownImage); - let imageDisplaySize = { - width: Math.round(maxWidth), - height: Math.round(maxHeight), - }; - if (imageNaturalSize) { const widthScale = maxWidth / imageNaturalSize.width; const heightScale = maxHeight / imageNaturalSize.height; const scale = Math.min(widthScale, heightScale); - imageDisplaySize = { + + return { width: Math.max(1, Math.round(imageNaturalSize.width * scale)), height: Math.max(1, Math.round(imageNaturalSize.height * scale)), }; - } + }, [imageNaturalSize, isMobile, viewport.height, viewport.width]); - const handleImageLoad = React.useCallback((event: React.SyntheticEvent) => { - const element = event.currentTarget; - const width = element.naturalWidth; - const height = element.naturalHeight; - if (width <= 0 || height <= 0) return; - - const url = element.getAttribute('src') ?? ''; - setLoadedImageSize((previous) => { - if (previous && previous.url === url && previous.width === width && previous.height === height) { - return previous; - } - return { url, width, height }; - }); - }, []); - - if (!currentImage) { + if (!isRendered || !currentImage || typeof document === 'undefined') { return null; } - const dialogLayout = getImagePreviewDialogLayout(imageDisplaySize, viewport, isMobile); - - return ( - - + +
+
); + + return createPortal(content, document.body); }; // ── PERF-007: Virtualised sub-components for dialog ────────────────── diff --git a/packages/ui/src/components/chat/message/imagePreviewSizing.ts b/packages/ui/src/components/chat/message/imagePreviewSizing.ts deleted file mode 100644 index 9ef842ee..00000000 --- a/packages/ui/src/components/chat/message/imagePreviewSizing.ts +++ /dev/null @@ -1,36 +0,0 @@ -export type ImagePreviewViewport = { width: number; height: number }; -type ImagePreviewSize = { width: number; height: number }; - -export const getImagePreviewBounds = ( - viewport: ImagePreviewViewport, - isMobile: boolean, - markdownImage: boolean, -): { maxWidth: number; maxHeight: number } => ({ - maxWidth: Math.max(160, viewport.width * (markdownImage ? 0.6 : (isMobile ? 0.86 : 0.75))), - maxHeight: Math.max(160, viewport.height * (markdownImage ? 0.8 : (isMobile ? 0.72 : 0.75))), -}); - -const IMAGE_DIALOG_MIN_WIDTH = 320; -const IMAGE_DIALOG_CHROME_WIDTH = 34; - -export const getImagePreviewDialogLayout = ( - image: ImagePreviewSize, - viewport: ImagePreviewViewport, - isMobile: boolean, -): { dialogWidth: number; imageWidth: number; imageHeight: number } => { - const viewportInset = isMobile ? 16 : 32; - const maxDialogWidth = Math.max(160, viewport.width - viewportInset); - const minDialogWidth = Math.min(IMAGE_DIALOG_MIN_WIDTH, maxDialogWidth); - const dialogWidth = Math.min( - maxDialogWidth, - Math.max(minDialogWidth, image.width + IMAGE_DIALOG_CHROME_WIDTH), - ); - const availableImageWidth = Math.max(1, dialogWidth - IMAGE_DIALOG_CHROME_WIDTH); - const scale = Math.min(1, availableImageWidth / Math.max(1, image.width)); - - return { - dialogWidth: Math.round(dialogWidth), - imageWidth: Math.max(1, Math.round(image.width * scale)), - imageHeight: Math.max(1, Math.round(image.height * scale)), - }; -}; diff --git a/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md b/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md index f709d590..d5215518 100644 --- a/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md +++ b/packages/ui/src/components/chat/message/parts/DOCUMENTATION.md @@ -63,14 +63,13 @@ Use this doc when you ask an agent to change tool/header/description behavior. are limited to 10 MiB, validated as PNG/JPEG/GIF/WebP, and local paths are fetched through the active runtime before conversion to data URLs. Local Markdown links whose target has one of - those image suffixes stay links in the text and open the same standard modal - preview as the gallery; image syntax does not insert a large inline image. A + those image suffixes stay links in the text and open the same existing + full-screen image preview as the gallery; image syntax does not insert a + large inline image. A completed assistant message hydrates at most 12 unique image candidates, including persisted text parts that omit their optional part-level end time. - The image modal reserves readable title width even for narrow portrait media. - This gallery and modal-preview enhancement is desktop-only. Mobile keeps the - standard Markdown rendering path until a dedicated, non-modal full-screen - image viewer can own safe areas, orientation, and touch gestures coherently. + Gallery clicks do not introduce or alter preview chrome: desktop and mobile + both reuse the pre-existing attachment image preview overlay. - `read` and `skill` are **static navigation tools** and render via `StaticToolRow`. - Every other tool, including search/fetch, OpenCode built-ins, custom tools, plugins, and MCP tools, is **expandable** and renders through `ToolPart`. - The managed `openchamber` plugin tool uses the expandable path and hides its broad protocol input. The plugin supplies the selected action's human description as the native tool title; the UI renders that metadata without owning an action map. The full versioned result envelope renders through the same neutral JSON summary/tree/raw views as other tools, without a tool-specific output card.