diff --git a/packages/ui/src/components/chat/MarkdownImageGallery.tsx b/packages/ui/src/components/chat/MarkdownImageGallery.tsx new file mode 100644 index 00000000..bdb2aa27 --- /dev/null +++ b/packages/ui/src/components/chat/MarkdownImageGallery.tsx @@ -0,0 +1,115 @@ +import React from 'react'; +import { Icon } from '@/components/icon/Icon'; +import { useEffectiveDirectory } from '@/hooks/useEffectiveDirectory'; +import type { ToolPopupContent } from './message/types'; +import { + extractMarkdownImageCandidates, + MAX_MARKDOWN_IMAGE_COUNT, + type MarkdownImageCandidate, +} from './markdown/markdownCore'; +import { resolveMarkdownImageSource } from './markdown/markdownImageAssets'; + +const MarkdownImageThumbnail: React.FC<{ + candidate: MarkdownImageCandidate; + directory: string; + onShowPopup?: (content: ToolPopupContent) => void; +}> = ({ candidate, directory, onShowPopup }) => { + const [image, setImage] = React.useState<{ + url: string; + status: 'loading' | 'ready' | 'error'; + }>({ url: '', status: 'loading' }); + + React.useEffect(() => { + const controller = new AbortController(); + setImage({ url: '', status: 'loading' }); + void resolveMarkdownImageSource(candidate.source, directory, controller.signal) + .then((url) => { + if (!controller.signal.aborted) setImage({ url, status: 'loading' }); + }) + .catch(() => { + if (!controller.signal.aborted) setImage({ url: '', status: 'error' }); + }); + return () => controller.abort(); + }, [candidate.source, directory]); + + const openPreview = React.useCallback(() => { + if (image.status !== 'ready' || !onShowPopup) return; + onShowPopup({ + open: true, + title: candidate.filename, + content: '', + metadata: { tool: 'markdown-image-preview', filename: candidate.filename }, + image: { url: image.url, filename: candidate.filename }, + }); + }, [candidate.filename, image, onShowPopup]); + + return ( + + ); +}; + +export const MarkdownImageGallery: React.FC<{ + contents: readonly string[]; + onShowPopup?: (content: ToolPopupContent) => void; +}> = ({ contents, onShowPopup }) => { + const directory = useEffectiveDirectory() ?? ''; + const candidates = React.useMemo( + () => extractMarkdownImageCandidates(contents, MAX_MARKDOWN_IMAGE_COUNT), + [contents], + ); + + if (candidates.length === 0) return null; + + return ( +
+ {candidates.map((candidate) => ( + + ))} +
+ ); +}; diff --git a/packages/ui/src/components/chat/MarkdownRenderer.tsx b/packages/ui/src/components/chat/MarkdownRenderer.tsx index c294bd6f..00c0e80a 100644 --- a/packages/ui/src/components/chat/MarkdownRenderer.tsx +++ b/packages/ui/src/components/chat/MarkdownRenderer.tsx @@ -17,6 +17,10 @@ const SimpleMarkdownRendererLazy = lazyWithChunkRecovery(() => loadMarkdownRendererModule().then((m) => ({ default: m.SimpleMarkdownRenderer })) ); +const MarkdownImageGalleryLazy = lazyWithChunkRecovery(() => + import('./MarkdownImageGallery').then((m) => ({ default: m.MarkdownImageGallery })) +); + const fallback =
; const fallbackContentClassName = (variant: unknown): string => { @@ -48,3 +52,9 @@ export const SimpleMarkdownRenderer: React.FC ); + +export const MarkdownImageGallery: React.FC> = (props) => ( + + + +); diff --git a/packages/ui/src/components/chat/MarkdownRendererImpl.tsx b/packages/ui/src/components/chat/MarkdownRendererImpl.tsx index 0a8bef4c..a56f165b 100644 --- a/packages/ui/src/components/chat/MarkdownRendererImpl.tsx +++ b/packages/ui/src/components/chat/MarkdownRendererImpl.tsx @@ -19,7 +19,8 @@ import { isDesktopLocalOriginActive, isDesktopShell, isVSCodeRuntime } from '@/l import { isMobileSurfaceRuntime } from '@/lib/runtimeSurface'; import { ensureOutsideFileGrantForDesktop } from '@/lib/outsideFileGrants'; import { getDirectoryForFilePath, isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils'; -import { renderMarkdownBlocks, renderMarkdownSync } from './markdown/markdownCore'; +import { getMarkdownImageFilename, renderMarkdownBlocks, renderMarkdownSync } from './markdown/markdownCore'; +import { resolveMarkdownImageSource } from './markdown/markdownImageAssets'; import { ensureMarkdownShikiTheme } from './markdown/markdownTheme'; import { getMarkdownSyntaxVars } from './markdown/markdownSyntaxVars'; import { @@ -107,6 +108,59 @@ const useExternalLinkInteractions = ({ }, [containerRef, enabled]); }; +const useMarkdownImageLinkInteractions = ({ + containerRef, + directory, + enabled, + onShowPopup, +}: { + containerRef: React.RefObject; + directory: string; + enabled: boolean; + onShowPopup?: (content: ToolPopupContent) => void; +}) => { + React.useEffect(() => { + const container = containerRef.current; + if (!enabled || !container || !onShowPopup) return; + + const controller = new AbortController(); + const handleClick = (event: MouseEvent) => { + if (event.defaultPrevented || event.button !== 0) return; + const target = event.target; + if (!(target instanceof Element)) return; + + const link = target.closest('[data-openchamber-markdown-image-link="true"]'); + if (!link || !container.contains(link)) return; + + const source = link.getAttribute('data-openchamber-markdown-image-source') ?? ''; + const filename = link.getAttribute('data-openchamber-markdown-image-filename') + || getMarkdownImageFilename(source, ''); + if (!source || !filename) return; + + event.preventDefault(); + event.stopPropagation(); + void resolveMarkdownImageSource(source, directory, controller.signal) + .then((url) => { + if (controller.signal.aborted || !link.isConnected) return; + onShowPopup({ + open: true, + title: filename, + content: '', + metadata: { tool: 'markdown-image-preview', filename }, + image: { url, filename }, + }); + }) + .catch(() => undefined); + }; + + container.addEventListener('click', handleClick); + return () => { + controller.abort(); + container.removeEventListener('click', handleClick); + }; + }, [containerRef, directory, enabled, onShowPopup]); +}; + const DEFAULT_MERMAID_CONTROLS: MermaidControlOptions = { download: true, copy: true, @@ -140,6 +194,7 @@ interface MarkdownRendererProps { variant?: MarkdownVariant; onShowPopup?: (content: ToolPopupContent) => void; enableFileReferences?: boolean; + enableLocalImages?: boolean; } const FILE_LINK_SELECTOR = '[data-openchamber-file-link="true"]'; @@ -497,6 +552,10 @@ const useFileReferenceInteractions = ({ let linkedCount = 0; for (const candidate of Array.from(candidates)) { + if (candidate.matches('[data-openchamber-markdown-image-link="true"]')) { + clearFileLinkAttributes(candidate); + continue; + } const rawCandidate = extractPathCandidateFromElement(candidate); const resolved = getResolvedReference(rawCandidate, effectiveDirectory); clearFileLinkAttributes(candidate); @@ -831,6 +890,7 @@ const useMorphdomMarkdown = ({ text, streaming, cacheKey, + deferImages = false, syntaxVars, ctx, }: { @@ -838,6 +898,7 @@ const useMorphdomMarkdown = ({ text: string; streaming: boolean; cacheKey: string; + deferImages?: boolean; syntaxVars: Record; ctx: DecorateContext; }) => { @@ -876,7 +937,7 @@ const useMorphdomMarkdown = ({ // `display:contents` keeps margin-collapsing/spacing identical to a flat // HTML body — the wrapper exists only for per-block reconciliation. block.style.display = 'contents'; - block.innerHTML = renderMarkdownSync(text); + block.innerHTML = renderMarkdownSync(text, deferImages); // Decorate synchronously too: wrap code blocks in their framed card, // mark inline code, build table controls, etc. The async pass re-decorates // its own DOM before morphing, so without this the first paint shows bare @@ -888,7 +949,7 @@ const useMorphdomMarkdown = ({ refreshMermaidViewers(); } } - }, [containerRef, text, ctx, refreshMermaidViewers]); + }, [containerRef, text, deferImages, ctx, refreshMermaidViewers]); React.useEffect(() => () => { mermaidViewerRef.current?.cleanup(); @@ -901,7 +962,7 @@ const useMorphdomMarkdown = ({ const target = container.querySelector('[data-markdown-content]') ?? container; let active = true; - void renderMarkdownBlocks(text, streaming, cacheKey).then((blocks) => { + void renderMarkdownBlocks(text, streaming, cacheKey, deferImages).then((blocks) => { if (!active) return; const existing = Array.from(target.children) as HTMLElement[]; @@ -952,7 +1013,7 @@ const useMorphdomMarkdown = ({ return () => { active = false; }; - }, [containerRef, text, streaming, cacheKey, ctx, refreshMermaidViewers]); + }, [containerRef, text, streaming, cacheKey, deferImages, ctx, refreshMermaidViewers]); React.useEffect(() => { const container = containerRef.current; @@ -999,6 +1060,7 @@ const MarkdownRendererImpl: React.FC = ({ variant = 'assistant', onShowPopup, enableFileReferences = true, + enableLocalImages = false, }) => { streamPerfCount('ui.markdown_renderer.render'); if (isStreaming) streamPerfCount('ui.markdown_renderer.render.streaming'); @@ -1030,15 +1092,33 @@ const MarkdownRendererImpl: React.FC = ({ enabled: enableFileReferences && !isStreaming, }); useExternalLinkInteractions({ containerRef }); + useMarkdownImageLinkInteractions({ + containerRef, + directory: effectiveDirectory, + enabled: enableLocalImages && variant === 'assistant' && !isStreaming, + onShowPopup, + }); const syntaxVars = React.useMemo(() => getMarkdownSyntaxVars(currentTheme), [currentTheme]); const ctx = useDecorateContext(currentTheme, live, effectiveDirectory ? handlePreviewLoopback : undefined, DEFAULT_MERMAID_CONTROLS); const cacheKey = `markdown-${part?.id ? `part-${part.id}` : `message-${messageId}`}`; - useMorphdomMarkdown({ containerRef, text: content, streaming: live, cacheKey, syntaxVars, ctx }); + useMorphdomMarkdown({ + containerRef, + text: content, + streaming: live, + cacheKey, + deferImages: enableLocalImages && variant === 'assistant' && !isStreaming, + syntaxVars, + ctx, + }); const markdownContent = ( -
+
); @@ -1065,6 +1145,7 @@ export const MarkdownRenderer = React.memo(MarkdownRendererImpl, (prev, next) => && prev.messageId === next.messageId && prev.onShowPopup === next.onShowPopup && prev.enableFileReferences === next.enableFileReferences + && prev.enableLocalImages === next.enableLocalImages && prev.part?.id === next.part?.id; }); diff --git a/packages/ui/src/components/chat/markdown/markdownCore.test.ts b/packages/ui/src/components/chat/markdown/markdownCore.test.ts index 86db0bbd..84bd6eba 100644 --- a/packages/ui/src/components/chat/markdown/markdownCore.test.ts +++ b/packages/ui/src/components/chat/markdown/markdownCore.test.ts @@ -1,7 +1,21 @@ -import { describe, expect, test } from 'bun:test'; +import { describe, expect, mock, test } from 'bun:test'; + +mock.module('dompurify', () => ({ + default: { + isSupported: true, + addHook: () => undefined, + sanitize: (html: string) => html, + }, +})); +mock.module('./markdown-worker', () => ({ + highlightCodeInWorker: async () => null, +})); import { escapeRawMarkdownHtml, MARKDOWN_FORBIDDEN_TAGS } from './markdownSecurity'; +const { extractMarkdownImageCandidates, renderMarkdownSync } = await import('./markdownCore'); +const { resolveMarkdownImageSource } = await import('./markdownImageAssets'); + describe('markdown sanitization', () => { test('turns raw assistant HTML into inert visible text', () => { const payload = ''; @@ -16,3 +30,102 @@ describe('markdown sanitization', () => { expect(MARKDOWN_FORBIDDEN_TAGS).toContain('style'); }); }); + +describe('Markdown images', () => { + test('keeps local image links in text and emits inert image placeholders', () => { + const html = renderMarkdownSync([ + '[linked image](packages/vscode/extension.jpg)', + '![image syntax](packages/vscode/extension.jpg)', + ].join('\n\n'), true); + + expect(html).toContain('data-openchamber-markdown-image-link="true"'); + expect(html.match(/data-openchamber-markdown-image-source="packages\/vscode\/extension.jpg"/g)).toHaveLength(1); + expect(html).toContain('data-openchamber-markdown-image-placeholder="true"'); + expect(html).toContain('image syntax'); + expect(html).not.toContain('src="packages/vscode/extension.jpg"'); + expect(html).not.toContain('data-openchamber-markdown-image-state'); + expect(html.match(/ { + const html = renderMarkdownSync([ + '[remote link](https://example.test/image.png)', + '![remote image](https://example.test/image.png)', + ].join('\n\n'), true); + + expect(html).toContain(' { + const html = renderMarkdownSync([ + '![file](file:///workspace/image.png)', + '![unsafe](javascript:alert(1))', + ].join('\n\n'), true); + + expect(html).toContain('role="img"'); + expect(html).not.toContain('src="file:'); + expect(html).not.toContain('src="javascript:'); + }); + + test('collects a single ordered gallery across mixed Markdown and ignores code', () => { + const candidates = extractMarkdownImageCandidates([ + [ + 'Before [local link](screens/first%20view.png) and `![code](ignored.png)`.', + '', + '- ![duplicate](screens/first%20view.png)', + '- ![remote](https://example.test/second.webp?size=2)', + '', + '```md', + '![fenced](ignored-too.jpg)', + '```', + ].join('\n'), + 'After ![third](data:image/png;base64,AAAA).', + ]); + + expect(candidates).toEqual([ + { source: 'screens/first%20view.png', filename: 'first view.png' }, + { source: 'https://example.test/second.webp?size=2', filename: 'second.webp' }, + { source: 'data:image/png;base64,AAAA', filename: 'third' }, + ]); + }); + + test('limits one finalized message gallery to twelve unique candidates', () => { + const markdown = Array.from({ length: 14 }, (_, index) => `![image ${index}](screens/${index}.png)`).join('\n'); + + const candidates = extractMarkdownImageCandidates([markdown]); + + expect(candidates).toHaveLength(12); + expect(candidates.at(-1)?.source).toBe('screens/11.png'); + }); + + test('validates embedded image bytes against the declared MIME type', async () => { + const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=='; + const signal = new AbortController().signal; + + expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, '', signal)).toBe(`data:image/png;base64,${png}`); + await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, '', signal).then( + () => { throw new Error('Expected mismatched image data to fail'); }, + (error: unknown) => expect((error as Error).message).toBe('Unsupported image data'), + ); + }); + + test('does not resolve images after cancellation', async () => { + const controller = new AbortController(); + controller.abort(); + + await resolveMarkdownImageSource('https://example.test/image.png', '', controller.signal).then( + () => { throw new Error('Expected an aborted image load to fail'); }, + (error: unknown) => expect((error as Error).name).toBe('AbortError'), + ); + }); + + test('keeps the existing image renderer outside finalized assistant text', () => { + const html = renderMarkdownSync('![tool image](https://example.test/image.png)'); + + expect(html).toContain(' value.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); +const LOCAL_IMAGE_EXTENSION_RE = /\.(?:png|jpe?g|gif|webp)(?:[?#].*)?$/i; +const WINDOWS_ABSOLUTE_PATH_RE = /^[A-Za-z]:[\\/]/; +const URL_SCHEME_RE = /^[A-Za-z][A-Za-z\d+.-]*:/; + +export interface MarkdownImageCandidate { + source: string; + filename: string; +} + +export const MAX_MARKDOWN_IMAGE_COUNT = 12; + +const isLocalMarkdownImageSource = (source: string): boolean => { + if (/^\/\//.test(source) || !LOCAL_IMAGE_EXTENSION_RE.test(source)) return false; + return WINDOWS_ABSOLUTE_PATH_RE.test(source) + || /^file:\/\//i.test(source) + || !URL_SCHEME_RE.test(source); +}; + +const isSupportedMarkdownImageSource = (source: string): boolean => ( + /^(?:https?:)?\/\//i.test(source) + || /^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source) + || isLocalMarkdownImageSource(source) +); + +export const getMarkdownImageFilename = (source: string, fallback: string): string => { + if (/^data:/i.test(source)) return fallback.trim(); + + const path = source.split(/[?#]/, 1)[0]?.replace(/\\/g, '/') ?? ''; + const encodedName = path.split('/').filter(Boolean).at(-1) ?? ''; + if (!encodedName) return fallback.trim(); + try { + return decodeURIComponent(encodedName); + } catch { + return encodedName; + } +}; + +export const extractMarkdownImageCandidates = ( + markdownTexts: readonly string[], + limit = MAX_MARKDOWN_IMAGE_COUNT, +): MarkdownImageCandidate[] => { + if (limit <= 0) return []; + + const candidates: MarkdownImageCandidate[] = []; + const seen = new Set(); + + for (const markdown of markdownTexts) { + if (!markdown || candidates.length >= limit) continue; + const tokens = marked.lexer(markdown); + marked.walkTokens(tokens, (token) => { + if (candidates.length >= limit) return; + + if (token.type !== 'image' && token.type !== 'link') return; + if (token.type === 'link' && !isLocalMarkdownImageSource(token.href ?? '')) return; + + const source = token.href ?? ''; + if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return; + const fallback = typeof token.text === 'string' ? token.text : ''; + const filename = getMarkdownImageFilename(source, fallback); + if (!filename) return; + + seen.add(source); + candidates.push({ source, filename }); + }); + } + + return candidates; +}; + +const renderMarkdownImage = ({ + href, + title, + text, +}: { + href: string; + title?: string | null; + text: string; +}): string => { + const source = href ?? ''; + const alt = escapeAttr(text ?? ''); + const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; + const supported = isSupportedMarkdownImageSource(source); + if (!supported) { + return `${alt}`; + } + + return `${alt}`; +}; + // --------------------------------------------------------------------------- // Streaming block segmentation (port of OpenCode's markdown-stream) // --------------------------------------------------------------------------- @@ -162,7 +251,7 @@ const blockMathExtension = { }, }; -const parser = marked.use({ +const createParser = (deferImages: boolean) => new Marked().use({ gfm: true, breaks: false, extensions: [inlineMathExtension, blockMathExtension], @@ -175,6 +264,11 @@ const parser = marked.use({ }, link({ href, title, text }) { const target = href ?? ''; + if (deferImages && isLocalMarkdownImageSource(target)) { + const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; + const filename = getMarkdownImageFilename(target, ''); + return `${text}`; + } const agentName = parseAgentHref(target); if (agentName) { return `${text}`; @@ -186,9 +280,13 @@ const parser = marked.use({ const titleAttr = title ? ` title="${escapeAttr(title)}"` : ''; return `${text}`; }, + ...(deferImages ? { image: renderMarkdownImage } : {}), }, }); +const parser = createParser(false); +const imageParser = createParser(true); + // --------------------------------------------------------------------------- // Math (KaTeX) — post-process the parsed HTML, skipping code/pre/kbd content // --------------------------------------------------------------------------- @@ -341,8 +439,8 @@ const touch = (key: string, entry: { hash: string; html: string }): void => { if (oldest) htmlCache.delete(oldest); }; -const parseBlock = async (block: MarkdownBlock): Promise => { - const parsed = await Promise.resolve(parser.parse(block.src)); +const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise => { + const parsed = await Promise.resolve((deferImages ? imageParser : parser).parse(block.src)); const withMath = renderMathExpressions(parsed); const highlighted = block.highlight ? await highlightCodeBlocks(withMath) : withMath; return sanitize(highlighted); @@ -357,9 +455,9 @@ const parseBlock = async (block: MarkdownBlock): Promise => { * is synchronous (marked is not configured `async`), so this never blocks on a * worker round-trip. */ -export const renderMarkdownSync = (text: string): string => { +export const renderMarkdownSync = (text: string, deferImages = false): string => { if (!text) return ''; - const parsed = parser.parse(text) as string; + const parsed = (deferImages ? imageParser : parser).parse(text) as string; const withMath = renderMathExpressions(parsed); return sanitize(withMath); }; @@ -382,6 +480,7 @@ export const renderMarkdownBlocks = async ( text: string, streaming: boolean, cacheKey: string, + deferImages = false, ): Promise => { if (!text) return []; @@ -389,14 +488,14 @@ export const renderMarkdownBlocks = async ( return Promise.all( blocks.map(async (block, index) => { const contentHash = hash(block.raw); - const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}`; - const key = `${cacheKey}:${index}:${block.mode}`; + const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${deferImages ? 1 : 0}`; + const key = `${cacheKey}:${index}:${block.mode}:${deferImages ? 1 : 0}`; const cached = htmlCache.get(key); if (cached && cached.hash === contentHash) { touch(key, cached); return { id, html: cached.html }; } - const html = await parseBlock(block); + const html = await parseBlock(block, deferImages); touch(key, { hash: contentHash, html }); return { id, html }; }), diff --git a/packages/ui/src/components/chat/markdown/markdownImageAssets.ts b/packages/ui/src/components/chat/markdown/markdownImageAssets.ts new file mode 100644 index 00000000..e61fea18 --- /dev/null +++ b/packages/ui/src/components/chat/markdown/markdownImageAssets.ts @@ -0,0 +1,132 @@ +import { isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils'; +import { runtimeFetch } from '@/lib/runtime-fetch'; + +const MAX_MARKDOWN_IMAGE_BYTES = 10 * 1024 * 1024; +const SUPPORTED_IMAGE_MIME_TYPES = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]); + +const parseLocalImagePath = (source: string): string => { + let value = source; + if (/^file:\/\//i.test(value)) { + try { + const fileUrl = new URL(value); + if (fileUrl.protocol !== 'file:') return ''; + value = fileUrl.host && fileUrl.host !== 'localhost' + ? `//${fileUrl.host}${fileUrl.pathname}` + : fileUrl.pathname; + if (/^\/[A-Za-z]:\//.test(value)) value = value.slice(1); + } catch { + return ''; + } + } + + const path = value.split(/[?#]/, 1)[0] ?? ''; + try { + return decodeURIComponent(path); + } catch { + return path; + } +}; + +const blobToDataUrl = (blob: Blob): Promise => new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => typeof reader.result === 'string' + ? resolve(reader.result) + : reject(new Error('Unable to encode image')); + reader.onerror = () => reject(reader.error ?? new Error('Unable to encode image')); + reader.readAsDataURL(blob); +}); + +const hasImageSignature = async (blob: Blob, mimeType: string): Promise => { + const bytes = new Uint8Array(await blob.slice(0, 12).arrayBuffer()); + const ascii = (start: number, end: number) => String.fromCharCode(...bytes.slice(start, end)); + if (mimeType === 'image/png') { + return bytes[0] === 0x89 && ascii(1, 4) === 'PNG' + && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a; + } + if (mimeType === 'image/jpeg') { + return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff; + } + if (mimeType === 'image/gif') { + const gif = ascii(0, 6); + return gif === 'GIF87a' || gif === 'GIF89a'; + } + return mimeType === 'image/webp' && ascii(0, 4) === 'RIFF' && ascii(8, 12) === 'WEBP'; +}; + +const validateImageBlob = async (blob: Blob, mimeType: string): Promise => { + if (!SUPPORTED_IMAGE_MIME_TYPES.has(mimeType)) throw new Error('Unsupported image type'); + if (blob.size > MAX_MARKDOWN_IMAGE_BYTES) throw new Error('Image is too large'); + if (!await hasImageSignature(blob, mimeType)) throw new Error('Unsupported image data'); +}; + +const validateDataImage = async (source: string): Promise => { + const match = /^data:(image\/(?:png|jpeg|gif|webp));base64,([\s\S]*)$/i.exec(source); + if (!match?.[1] || match[2] === undefined) throw new Error('Invalid image data URL'); + const encoded = match[2]; + if (encoded.length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) { + throw new Error('Image is too large'); + } + + let binary: string; + try { + binary = atob(encoded); + } catch { + throw new Error('Invalid image data URL'); + } + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index); + await validateImageBlob(new Blob([bytes]), match[1].toLowerCase()); +}; + +export const resolveMarkdownImageSource = async ( + source: string, + directory: string, + signal: AbortSignal, +): Promise => { + if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError'); + if (/^(?:https?:)?\/\//i.test(source)) return source; + + if (/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)) { + await validateDataImage(source); + if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError'); + return source; + } + + const localPath = parseLocalImagePath(source); + const absolutePath = toAbsoluteFilePath(directory, localPath); + if (!directory || !localPath || !isFilePathWithinDirectory(absolutePath, directory)) { + throw new Error('Image path is outside the active workspace'); + } + + const statResponse = await runtimeFetch('/api/fs/stat', { + query: { path: absolutePath, directory, optional: 'true' }, + signal, + }); + if (!statResponse.ok) throw new Error(`Unable to inspect image (${statResponse.status})`); + const stat = await statResponse.json() as { isFile?: boolean; size?: number }; + if (!stat.isFile) throw new Error('Image path is not a file'); + if (typeof stat.size === 'number' && stat.size > MAX_MARKDOWN_IMAGE_BYTES) { + throw new Error('Image is too large'); + } + + const response = await runtimeFetch('/api/fs/raw', { + query: { path: absolutePath, directory }, + signal, + }); + if (!response.ok) throw new Error(`Unable to load image (${response.status})`); + + const mimeType = (response.headers.get('content-type') ?? '').split(';', 1)[0]?.toLowerCase() ?? ''; + const contentLength = Number(response.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > MAX_MARKDOWN_IMAGE_BYTES) { + throw new Error('Image is too large'); + } + + const blob = await response.blob(); + await validateImageBlob(blob, mimeType); + return blobToDataUrl(blob); +}; diff --git a/packages/ui/src/components/chat/message/MessageBody.tsx b/packages/ui/src/components/chat/message/MessageBody.tsx index d2e04411..a697c165 100644 --- a/packages/ui/src/components/chat/message/MessageBody.tsx +++ b/packages/ui/src/components/chat/message/MessageBody.tsx @@ -21,7 +21,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip import { ArrowsMerge } from '@/components/icons/ArrowsMerge'; import type { ContentChangeReason } from '@/hooks/useChatAutoFollow'; -import { SimpleMarkdownRenderer } from '../MarkdownRenderer'; +import { MarkdownImageGallery, SimpleMarkdownRenderer } from '../MarkdownRenderer'; import { useSessionUIStore } from '@/sync/session-ui-store'; import { useUIStore } from '@/stores/useUIStore'; import { flattenAssistantTextParts, suggestPlanTitleFromText } from '@/lib/messages/messageText'; @@ -1211,6 +1211,11 @@ const AssistantMessageBody = React.memo(({ const assistantTextParts = React.useMemo(() => { return visibleParts.filter((part) => part.type === 'text'); }, [visibleParts]); + const finalizedAssistantMarkdownContents = React.useMemo(() => ( + isMessageCompleted + ? assistantTextParts.map(extractTextContent).filter((text) => text.trim().length > 0) + : [] + ), [assistantTextParts, isMessageCompleted]); const assistantPlanText = React.useMemo(() => flattenAssistantTextParts(assistantTextParts), [assistantTextParts]); const suggestedPlanTitle = React.useMemo(() => suggestPlanTitleFromText(assistantPlanText), [assistantPlanText]); @@ -1863,6 +1868,7 @@ const AssistantMessageBody = React.memo(({ chatRenderMode={chatRenderMode} onContentChange={onContentChange} onShowPopup={onShowPopup} + enableMarkdownImages={isMessageCompleted} />
); @@ -2017,6 +2023,7 @@ const AssistantMessageBody = React.memo(({ collapsedPreviewCount, expandedTools, isMobile, + isMessageCompleted, isActivityOwnerMessage, isSortedRenderMode, lastRenderableTextPartIndex, @@ -2228,6 +2235,10 @@ const AssistantMessageBody = React.memo(({ )}
+ {shouldRenderStandaloneActionsAfterContent && (
diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts index 08584eb8..a0d7f916 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test'; +import { getImagePreviewBounds, getImagePreviewDialogLayout } from './imagePreviewSizing'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; describe('getMermaidDataUrlSourcePromise', () => { @@ -29,3 +30,43 @@ describe('Mermaid load request ids', () => { expect(isCurrentMermaidLoadRequest(secondRequest, secondRequest)).toBe(true); }); }); + +describe('Markdown image preview bounds', () => { + test('uses sixty percent of the viewport width with vertical containment', () => { + expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, true)).toEqual({ + maxWidth: 720, + maxHeight: 640, + }); + }); + + test('preserves existing attachment preview bounds', () => { + expect(getImagePreviewBounds({ width: 1200, height: 800 }, false, false)).toEqual({ + maxWidth: 900, + maxHeight: 600, + }); + }); + + test('keeps a readable modal width for narrow portrait images', () => { + expect(getImagePreviewDialogLayout( + { width: 29, height: 576 }, + { width: 1280, height: 720 }, + false, + )).toEqual({ + dialogWidth: 320, + imageWidth: 29, + imageHeight: 576, + }); + }); + + test('fits image content inside the mobile dialog chrome without cropping', () => { + expect(getImagePreviewDialogLayout( + { width: 275, height: 500 }, + { width: 320, height: 700 }, + true, + )).toEqual({ + dialogWidth: 304, + imageWidth: 270, + imageHeight: 491, + }); + }); +}); diff --git a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx index 15298662..ce01e48b 100644 --- a/packages/ui/src/components/chat/message/ToolOutputDialog.tsx +++ b/packages/ui/src/components/chat/message/ToolOutputDialog.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Dialog, DialogContent } from '@/components/ui/dialog'; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; import { File as PierreFile, PatchDiff } from '@pierre/diffs/react'; import { WorkerHighlightedCode } from '@/components/code/WorkerHighlightedCode'; import { createPortal } from 'react-dom'; @@ -29,6 +29,11 @@ import { Icon } from "@/components/icon/Icon"; import { useI18n, type I18nKey, type I18nParams } from '@/lib/i18n'; import { runtimeFetch } from '@/lib/runtime-fetch'; import { MermaidLoadFailure, getMermaidDataUrlSourcePromise, isCurrentMermaidLoadRequest, isMermaidLoadFailure, nextMermaidLoadRequestId } from './toolOutputDialogMermaid'; +import { + getImagePreviewBounds, + getImagePreviewDialogLayout, + type ImagePreviewViewport, +} from './imagePreviewSizing'; interface ToolOutputDialogProps { popup: ToolPopupContent; @@ -158,7 +163,7 @@ const usePierreThemeConfig = (): PierreThemeConfig => { }; }; -type ViewportSize = { width: number; height: number }; +type ViewportSize = ImagePreviewViewport; const getWindowViewport = (): ViewportSize => ({ width: typeof window !== 'undefined' ? window.innerWidth : 0, @@ -331,8 +336,11 @@ const ImagePreviewDialog: React.FC<{ }, [popup.image]); const [currentIndex, setCurrentIndex] = React.useState(0); - const [imageNaturalSize, setImageNaturalSize] = React.useState<{ width: number; height: number } | null>(null); - const { isRendered, isVisible, isTransitioning } = usePreviewOverlayState(popup.open); + const [loadedImageSize, setLoadedImageSize] = React.useState<{ + url: string; + width: number; + height: number; + } | null>(null); const viewport = usePreviewViewport(popup.open); React.useEffect(() => { @@ -352,9 +360,10 @@ const ImagePreviewDialog: React.FC<{ setCurrentIndex(matchingIndex >= 0 ? matchingIndex : 0); }, [gallery, popup.image?.index, popup.image?.url, popup.open]); - const currentImage = gallery[currentIndex] ?? gallery[0] ?? popup.image; + const currentImage = gallery[currentIndex] ?? gallery[0]; const imageTitle = currentImage?.filename || popup.title || 'Image preview'; const hasMultipleImages = gallery.length > 1; + const markdownImage = popup.metadata?.tool === 'markdown-image-preview'; const showPrevious = React.useCallback(() => { if (gallery.length <= 1) return; @@ -372,11 +381,6 @@ const ImagePreviewDialog: React.FC<{ } const onKeyDown = (event: KeyboardEvent) => { - if (event.key === 'Escape') { - onOpenChange(false); - return; - } - if (event.key === 'ArrowLeft' && hasMultipleImages) { event.preventDefault(); showPrevious(); @@ -393,126 +397,108 @@ const ImagePreviewDialog: React.FC<{ return () => { window.removeEventListener('keydown', onKeyDown); }; - }, [hasMultipleImages, onOpenChange, popup.open, showNext, showPrevious]); + }, [hasMultipleImages, popup.open, showNext, showPrevious]); - React.useEffect(() => { - setImageNaturalSize(null); - }, [currentImage?.url]); - - const imageDisplaySize = React.useMemo(() => { - const maxWidth = Math.max(160, viewport.width * (isMobile ? 0.86 : 0.75)); - const maxHeight = Math.max(160, viewport.height * (isMobile ? 0.72 : 0.75)); - - if (!imageNaturalSize) { - return { - width: Math.round(maxWidth), - height: Math.round(maxHeight), - }; - } + const imageNaturalSize = loadedImageSize?.url === currentImage?.url + ? loadedImageSize + : null; + const { maxWidth, maxHeight } = getImagePreviewBounds(viewport, isMobile, markdownImage); + let imageDisplaySize = { + width: Math.round(maxWidth), + height: Math.round(maxHeight), + }; + if (imageNaturalSize) { const widthScale = maxWidth / imageNaturalSize.width; const heightScale = maxHeight / imageNaturalSize.height; const scale = Math.min(widthScale, heightScale); - - return { + imageDisplaySize = { width: Math.max(1, Math.round(imageNaturalSize.width * scale)), height: Math.max(1, Math.round(imageNaturalSize.height * scale)), }; - }, [imageNaturalSize, isMobile, viewport.height, viewport.width]); + } - if (!isRendered || !currentImage || typeof document === 'undefined') { + const handleImageLoad = React.useCallback((event: React.SyntheticEvent) => { + const element = event.currentTarget; + const width = element.naturalWidth; + const height = element.naturalHeight; + if (width <= 0 || height <= 0) return; + + const url = element.getAttribute('src') ?? ''; + setLoadedImageSize((previous) => { + if (previous && previous.url === url && previous.width === width && previous.height === height) { + return previous; + } + return { url, width, height }; + }); + }, []); + + if (!currentImage) { return null; } - const content = ( -
-