fix(markdown): correct image gallery rendering (#2894)

This commit is contained in:
ChangeHow
2026-08-14 17:10:36 +03:00
committed by GitHub
parent fe1f6130d6
commit 90780258cd
18 changed files with 1022 additions and 284 deletions
@@ -57,6 +57,7 @@ const ICONS = {
zoomOut: spriteIcon('subtract'),
fit: spriteIcon('refresh'),
textWrap: spriteIcon('text-wrap'),
image: spriteIcon('file-image'),
} as const;
const ICON_BTN_CLASS =
@@ -66,6 +67,18 @@ const setIconHtml = (el: Element, html: string): void => {
el.innerHTML = html;
};
const decorateImageLabels = (root: HTMLElement): void => {
for (const label of Array.from(root.querySelectorAll<HTMLElement>('[data-openchamber-markdown-image-label="true"]'))) {
if (label.querySelector('[data-openchamber-markdown-image-label-icon]')) continue;
const icon = document.createElement('span');
icon.className = 'inline-flex shrink-0';
icon.setAttribute('aria-hidden', 'true');
icon.setAttribute('data-openchamber-markdown-image-label-icon', 'true');
setIconHtml(icon, ICONS.image);
label.prepend(icon);
}
};
const makeIconButton = (icon: keyof typeof ICONS, title: string, slot: string): HTMLButtonElement => {
const button = document.createElement('button');
button.type = 'button';
@@ -487,6 +500,7 @@ const decorateLinks = (root: HTMLElement, ctx: DecorateContext): void => {
/** Run all idempotent DOM decoration passes over freshly-rendered markdown. */
export const decorateMarkdown = (root: HTMLElement, ctx: DecorateContext): void => {
decorateImageLabels(root);
decorateInlineCode(root);
decorateMermaid(root, ctx);
decorateCodeBlocks(root, ctx);
@@ -13,7 +13,11 @@ mock.module('./markdown-worker', () => ({
import { escapeRawMarkdownHtml, isLocalFileUrl, MARKDOWN_FORBIDDEN_TAGS } from './markdownSecurity';
const { extractMarkdownImageCandidates, renderMarkdownSync } = await import('./markdownCore');
const {
__markdownImageCandidateCacheForTests,
extractMarkdownImageCandidates,
renderMarkdownSync,
} = await import('./markdownCore');
const { resolveMarkdownImageSource } = await import('./markdownImageAssets');
describe('markdown sanitization', () => {
@@ -39,45 +43,31 @@ describe('markdown sanitization', () => {
});
describe('Markdown images', () => {
test('keeps local image links in text and emits inert image placeholders', () => {
test('renders assistant images as icon-ready text without loading the source', () => {
const html = renderMarkdownSync([
'[linked image](packages/vscode/extension.jpg)',
'![image syntax](packages/vscode/extension.jpg)',
].join('\n\n'), true);
].join('\n\n'), 'label');
expect(html).toContain('data-openchamber-markdown-image-link="true"');
expect(html.match(/data-openchamber-markdown-image-source="packages\/vscode\/extension.jpg"/g)).toHaveLength(1);
expect(html).toContain('data-openchamber-markdown-image-placeholder="true"');
expect(html).toContain('image syntax');
expect(html).not.toContain('src="packages/vscode/extension.jpg"');
expect(html).not.toContain('data-openchamber-markdown-image-state');
expect(html).toContain('data-openchamber-markdown-image-label="true"');
expect(html).toContain('extension.jpg');
expect(html).not.toContain('image syntax');
expect(html).not.toContain('<img');
expect(html.match(/<a /g)).toHaveLength(1);
});
test('keeps HTTP links as links and defers remote image tokens to finalized rendering', () => {
test('keeps non-chat Markdown images inline', () => {
const html = renderMarkdownSync([
'[remote link](https://example.test/image.png)',
'![remote image](https://example.test/image.png)',
].join('\n\n'), true);
].join('\n\n'));
expect(html).toContain('<a href="https://example.test/image.png"');
expect(html).not.toContain('<img');
expect(html).toContain('data-openchamber-markdown-image-placeholder="true"');
expect(html).toContain('remote image');
expect(html).toContain('<img src="https://example.test/image.png" alt="remote image">');
expect(html).not.toContain('data-openchamber-markdown-image-label');
});
test('preserves file URLs inertly and never activates unknown schemes', () => {
const html = renderMarkdownSync([
'![file](file:///workspace/image.png)',
'![unsafe](javascript:alert(1))',
].join('\n\n'), true);
expect(html).toContain('role="img"');
expect(html).not.toContain('src="file:');
expect(html).not.toContain('src="javascript:');
});
test('collects a single ordered gallery across mixed Markdown and ignores code', () => {
test('collects image syntax across mixed Markdown and ignores links and code', () => {
const candidates = extractMarkdownImageCandidates([
[
'Before [local link](screens/first%20view.png) and `![code](ignored.png)`.',
@@ -99,6 +89,10 @@ describe('Markdown images', () => {
]);
});
test('does not add an ordinary local image link to the gallery', () => {
expect(extractMarkdownImageCandidates(['[download](screens/image.png)'])).toEqual([]);
});
test('limits one finalized message gallery to twelve unique candidates', () => {
const markdown = Array.from({ length: 14 }, (_, index) => `![image ${index}](screens/${index}.png)`).join('\n');
@@ -108,12 +102,69 @@ describe('Markdown images', () => {
expect(candidates.at(-1)?.source).toBe('screens/11.png');
});
test('reuses extracted candidates across virtualized remounts without changing gallery behavior', () => {
__markdownImageCandidateCacheForTests.reset();
const contents = Array.from({ length: 20 }, (_, index) => `![image ${index}](screens/${index}.png)`);
expect(extractMarkdownImageCandidates(contents)).toHaveLength(12);
expect(__markdownImageCandidateCacheForTests.stats().scans).toBe(12);
for (let round = 0; round < 1000; round += 1) {
expect(extractMarkdownImageCandidates(contents)).toHaveLength(12);
}
const stats = __markdownImageCandidateCacheForTests.stats();
expect(stats.entries).toBe(12);
expect(stats.scans).toBe(12);
});
test('scans one thousand independent messages once across virtualized remounts', () => {
__markdownImageCandidateCacheForTests.reset();
const messages = Array.from(
{ length: 1000 },
(_, index) => `![image ${index}](screens/${index}.png)`,
);
for (const message of messages) extractMarkdownImageCandidates([message]);
for (const message of messages) extractMarkdownImageCandidates([message]);
const stats = __markdownImageCandidateCacheForTests.stats();
expect(stats.entries).toBe(1000);
expect(stats.scans).toBe(1000);
});
test('gives embedded images without alt text a stable filename', () => {
const source = 'data:image/png;base64,AAAA';
expect(extractMarkdownImageCandidates([`![](${source})`])).toEqual([
{ source, filename: 'image.png' },
]);
expect(renderMarkdownSync(`![](${source})`, 'label')).toContain('image.png');
});
test('bounds cached candidate entries and bytes, and skips oversized individual content', () => {
__markdownImageCandidateCacheForTests.reset();
for (let index = 0; index < 1025; index += 1) {
extractMarkdownImageCandidates([`![image ${index}](screens/${index}.png)`]);
}
const boundedStats = __markdownImageCandidateCacheForTests.stats();
expect(boundedStats.entries).toBe(1024);
expect(boundedStats.bytes <= 2 * 1024 * 1024).toBe(true);
__markdownImageCandidateCacheForTests.reset();
const oversized = `![image](screens/large.png)\n${'x'.repeat(64 * 1024)}`;
extractMarkdownImageCandidates([oversized]);
extractMarkdownImageCandidates([oversized]);
expect(__markdownImageCandidateCacheForTests.stats()).toEqual({ entries: 0, bytes: 0, scans: 2 });
});
test('validates embedded image bytes against the declared MIME type', async () => {
const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==';
const signal = new AbortController().signal;
expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, '', signal)).toBe(`data:image/png;base64,${png}`);
await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, '', signal).then(
expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, signal)).toBe(`data:image/png;base64,${png}`);
await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, signal).then(
() => { throw new Error('Expected mismatched image data to fail'); },
(error: unknown) => expect((error as Error).message).toBe('Unsupported image data'),
);
@@ -123,7 +174,7 @@ describe('Markdown images', () => {
const controller = new AbortController();
controller.abort();
await resolveMarkdownImageSource('https://example.test/image.png', '', controller.signal).then(
await resolveMarkdownImageSource('https://example.test/image.png', controller.signal).then(
() => { throw new Error('Expected an aborted image load to fail'); },
(error: unknown) => expect((error as Error).name).toBe('AbortError'),
);
@@ -133,6 +184,6 @@ describe('Markdown images', () => {
const html = renderMarkdownSync('![tool image](https://example.test/image.png)');
expect(html).toContain('<img src="https://example.test/image.png"');
expect(html).not.toContain('data-openchamber-markdown-image-placeholder');
expect(html).not.toContain('data-openchamber-markdown-image');
});
});
@@ -19,8 +19,23 @@ export interface MarkdownImageCandidate {
filename: string;
}
export type MarkdownImageMode = 'inline' | 'label';
export const MAX_MARKDOWN_IMAGE_COUNT = 12;
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRIES = 1024;
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_BYTES = 2 * 1024 * 1024;
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRY_BYTES = 64 * 1024;
type MarkdownImageCandidateCacheEntry = {
candidates: MarkdownImageCandidate[];
bytes: number;
};
const markdownImageCandidateCache = new Map<string, MarkdownImageCandidateCacheEntry>();
let markdownImageCandidateCacheBytes = 0;
let markdownImageCandidateScanCount = 0;
const isLocalMarkdownImageSource = (source: string): boolean => {
if (/^\/\//.test(source) || !LOCAL_IMAGE_EXTENSION_RE.test(source)) return false;
return WINDOWS_ABSOLUTE_PATH_RE.test(source)
@@ -34,8 +49,11 @@ const isSupportedMarkdownImageSource = (source: string): boolean => (
|| isLocalMarkdownImageSource(source)
);
export const getMarkdownImageFilename = (source: string, fallback: string): string => {
if (/^data:/i.test(source)) return fallback.trim();
const getMarkdownImageFilename = (source: string, fallback: string): string => {
if (/^data:image\/(png|jpeg|gif|webp)/i.test(source)) {
const extension = /^data:image\/([^;,]+)/i.exec(source)?.[1]?.replace('jpeg', 'jpg') ?? 'png';
return fallback.trim() || `image.${extension}`;
}
const path = source.split(/[?#]/, 1)[0]?.replace(/\\/g, '/') ?? '';
const encodedName = path.split('/').filter(Boolean).at(-1) ?? '';
@@ -47,6 +65,87 @@ export const getMarkdownImageFilename = (source: string, fallback: string): stri
}
};
const estimateMarkdownImageCandidateCacheEntryBytes = (
markdown: string,
candidates: readonly MarkdownImageCandidate[],
): number => (
(markdown.length + candidates.reduce((total, candidate) => total + candidate.source.length + candidate.filename.length, 0)) * 2
);
const scanMarkdownImageCandidates = (markdown: string): MarkdownImageCandidate[] => {
markdownImageCandidateScanCount += 1;
const candidates: MarkdownImageCandidate[] = [];
const seen = new Set<string>();
const tokens = marked.lexer(markdown);
marked.walkTokens(tokens, (token) => {
if (token.type !== 'image') return;
const source = token.href ?? '';
if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return;
const fallback = typeof token.text === 'string' ? token.text : '';
const filename = getMarkdownImageFilename(source, fallback);
if (!filename) return;
seen.add(source);
candidates.push({ source, filename });
});
return candidates;
};
const getMarkdownImageCandidates = (markdown: string): MarkdownImageCandidate[] => {
const cached = markdownImageCandidateCache.get(markdown);
if (cached) {
markdownImageCandidateCache.delete(markdown);
markdownImageCandidateCache.set(markdown, cached);
return cached.candidates;
}
const candidates = scanMarkdownImageCandidates(markdown);
const bytes = estimateMarkdownImageCandidateCacheEntryBytes(markdown, candidates);
if (bytes > MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRY_BYTES) return candidates;
while (
markdownImageCandidateCache.size >= MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRIES
|| markdownImageCandidateCacheBytes + bytes > MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_BYTES
) {
const oldest = markdownImageCandidateCache.entries().next().value;
if (!oldest) break;
markdownImageCandidateCache.delete(oldest[0]);
markdownImageCandidateCacheBytes -= oldest[1].bytes;
}
markdownImageCandidateCache.set(markdown, { candidates, bytes });
markdownImageCandidateCacheBytes += bytes;
return candidates;
};
/** @internal Test-only cache instrumentation for deterministic regression tests. */
export const __markdownImageCandidateCacheForTests = {
reset: (): void => {
markdownImageCandidateCache.clear();
markdownImageCandidateCacheBytes = 0;
markdownImageCandidateScanCount = 0;
},
stats: () => ({
entries: markdownImageCandidateCache.size,
bytes: markdownImageCandidateCacheBytes,
scans: markdownImageCandidateScanCount,
}),
};
const renderMarkdownImageLabel = ({
href,
title,
text,
}: {
href: string;
title?: string | null;
text: string;
}): string => {
const label = getMarkdownImageFilename(href ?? '', text);
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
return `<span${titleAttr} class="inline-flex items-center gap-1 align-text-bottom text-muted-foreground" data-openchamber-markdown-image-label="true">${escapeAttr(label)}</span>`;
};
export const extractMarkdownImageCandidates = (
markdownTexts: readonly string[],
limit = MAX_MARKDOWN_IMAGE_COUNT,
@@ -58,47 +157,17 @@ export const extractMarkdownImageCandidates = (
for (const markdown of markdownTexts) {
if (!markdown || candidates.length >= limit) continue;
const tokens = marked.lexer(markdown);
marked.walkTokens(tokens, (token) => {
if (candidates.length >= limit) return;
if (token.type !== 'image' && token.type !== 'link') return;
if (token.type === 'link' && !isLocalMarkdownImageSource(token.href ?? '')) return;
const source = token.href ?? '';
if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return;
const fallback = typeof token.text === 'string' ? token.text : '';
const filename = getMarkdownImageFilename(source, fallback);
if (!filename) return;
seen.add(source);
candidates.push({ source, filename });
});
for (const candidate of getMarkdownImageCandidates(markdown)) {
if (candidates.length >= limit) break;
if (seen.has(candidate.source)) continue;
seen.add(candidate.source);
candidates.push({ ...candidate });
}
}
return candidates;
};
const renderMarkdownImage = ({
href,
title,
text,
}: {
href: string;
title?: string | null;
text: string;
}): string => {
const source = href ?? '';
const alt = escapeAttr(text ?? '');
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
const supported = isSupportedMarkdownImageSource(source);
if (!supported) {
return `<span role="img" aria-label="${alt}"${titleAttr}>${alt}</span>`;
}
return `<span role="img" aria-label="${alt}"${titleAttr} data-openchamber-markdown-image-placeholder="true">${alt}</span>`;
};
// ---------------------------------------------------------------------------
// Streaming block segmentation (port of OpenCode's markdown-stream)
// ---------------------------------------------------------------------------
@@ -251,7 +320,7 @@ const blockMathExtension = {
},
};
const createParser = (deferImages: boolean) => new Marked().use({
const createParser = (imageMode: MarkdownImageMode) => new Marked().use({
gfm: true,
breaks: false,
extensions: [inlineMathExtension, blockMathExtension],
@@ -264,11 +333,6 @@ const createParser = (deferImages: boolean) => new Marked().use({
},
link({ href, title, text }) {
const target = href ?? '';
if (deferImages && isLocalMarkdownImageSource(target)) {
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
const filename = getMarkdownImageFilename(target, '');
return `<a href="${escapeAttr(target)}"${titleAttr} class="external-link" target="_blank" rel="noopener noreferrer" data-openchamber-markdown-image-link="true" data-openchamber-markdown-image-source="${escapeAttr(target)}" data-openchamber-markdown-image-filename="${escapeAttr(filename)}">${text}</a>`;
}
const agentName = parseAgentHref(target);
if (agentName) {
return `<a href="${escapeAttr(buildAgentMentionUrl(agentName))}" data-openchamber-agent-mention="true" class="text-primary hover:underline" target="_blank" rel="noopener noreferrer">${text}</a>`;
@@ -280,12 +344,12 @@ const createParser = (deferImages: boolean) => new Marked().use({
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
return `<a href="${escapeAttr(target)}"${titleAttr} class="external-link" target="_blank" rel="noopener noreferrer">${text}</a>`;
},
...(deferImages ? { image: renderMarkdownImage } : {}),
...(imageMode === 'label' ? { image: renderMarkdownImageLabel } : {}),
},
});
const parser = createParser(false);
const imageParser = createParser(true);
const inlineImageParser = createParser('inline');
const imageLabelParser = createParser('label');
// ---------------------------------------------------------------------------
// Math (KaTeX) — post-process the parsed HTML, skipping code/pre/kbd content
@@ -443,8 +507,9 @@ const touch = (key: string, entry: { hash: string; html: string }): void => {
if (oldest) htmlCache.delete(oldest);
};
const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise<string> => {
const parsed = await Promise.resolve((deferImages ? imageParser : parser).parse(block.src));
const parseBlock = async (block: MarkdownBlock, imageMode: MarkdownImageMode): Promise<string> => {
const parser = imageMode === 'label' ? imageLabelParser : inlineImageParser;
const parsed = await Promise.resolve(parser.parse(block.src));
const withMath = renderMathExpressions(parsed);
const highlighted = block.highlight ? await highlightCodeBlocks(withMath) : withMath;
return sanitize(highlighted);
@@ -459,9 +524,10 @@ const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise<s
* is synchronous (marked is not configured `async`), so this never blocks on a
* worker round-trip.
*/
export const renderMarkdownSync = (text: string, deferImages = false): string => {
export const renderMarkdownSync = (text: string, imageMode: MarkdownImageMode = 'inline'): string => {
if (!text) return '';
const parsed = (deferImages ? imageParser : parser).parse(text) as string;
const parser = imageMode === 'label' ? imageLabelParser : inlineImageParser;
const parsed = parser.parse(text) as string;
const withMath = renderMathExpressions(parsed);
return sanitize(withMath);
};
@@ -484,7 +550,7 @@ export const renderMarkdownBlocks = async (
text: string,
streaming: boolean,
cacheKey: string,
deferImages = false,
imageMode: MarkdownImageMode = 'inline',
): Promise<RenderedBlock[]> => {
if (!text) return [];
@@ -492,14 +558,14 @@ export const renderMarkdownBlocks = async (
return Promise.all(
blocks.map(async (block, index) => {
const contentHash = hash(block.raw);
const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${deferImages ? 1 : 0}`;
const key = `${cacheKey}:${index}:${block.mode}:${deferImages ? 1 : 0}`;
const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${imageMode}`;
const key = `${cacheKey}:${index}:${block.mode}:${imageMode}`;
const cached = htmlCache.get(key);
if (cached && cached.hash === contentHash) {
touch(key, cached);
return { id, html: cached.html };
}
const html = await parseBlock(block, deferImages);
const html = await parseBlock(block, imageMode);
touch(key, { hash: contentHash, html });
return { id, html };
}),
@@ -0,0 +1,68 @@
import { describe, expect, mock, test } from 'bun:test';
let requestCount = 0;
const runtimeFetch = mock(async (_path: string, init?: RequestInit) => {
requestCount += 1;
const body = JSON.parse(String(init?.body)) as { sources: string[] };
return new Response(JSON.stringify({
results: body.sources.map((source) => ({ source, status: 'ready', path: `/repo/${source}` })),
}), { status: 200, headers: { 'content-type': 'application/json' } });
});
const resolver = {
api: () => '',
authenticatedAsset: (path: string, query: Record<string, string | undefined>) => {
const params = new URLSearchParams(Object.entries(query).filter((entry): entry is [string, string] => Boolean(entry[1])));
return `${path}?${params}`;
},
};
mock.module('@/lib/runtime-fetch', () => ({ runtimeFetch }));
mock.module('@/lib/runtime-url', () => ({ getRuntimeUrlResolver: () => resolver }));
const { getPreparedMarkdownImageUrl, prepareLocalMarkdownImages } = await import('./markdownImageAssets');
describe('Markdown image asset preparation', () => {
test('prepares many images in one message-level request', async () => {
requestCount = 0;
const sources = Array.from({ length: 12 }, (_, index) => `${index}.png`);
const result = await prepareLocalMarkdownImages({
sources,
directory: '/repo',
sessionId: 'ses_batch',
messageId: 'msg_batch',
signal: new AbortController().signal,
});
expect(result.size).toBe(12);
expect(requestCount).toBe(1);
});
test('reuses preparation for one thousand messages after virtualized remounts', async () => {
requestCount = 0;
const requests = Array.from({ length: 1000 }, (_, index) => ({
sources: [`${index}.png`],
directory: '/repo',
sessionId: 'ses_long',
messageId: `msg_${index}`,
signal: new AbortController().signal,
}));
for (const request of requests) await prepareLocalMarkdownImages(request);
for (const request of requests) await prepareLocalMarkdownImages(request);
expect(requestCount).toBe(1000);
});
test('reuses the existing authenticated raw-file asset URL', () => {
const url = getPreparedMarkdownImageUrl({
status: 'ready',
path: '/tmp/opencode/image.png',
outsideFileGrant: 'grant-1',
}, '/repo');
expect(url).toContain('/api/fs/raw?');
expect(url).toContain('path=%2Ftmp%2Fopencode%2Fimage.png');
expect(url).toContain('outsideFileGrant=grant-1');
});
});
@@ -1,7 +1,9 @@
import { isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils';
import { runtimeFetch } from '@/lib/runtime-fetch';
import { getRuntimeUrlResolver, type RuntimeUrlResolver } from '@/lib/runtime-url';
const MAX_MARKDOWN_IMAGE_BYTES = 10 * 1024 * 1024;
const MAX_PREPARE_CACHE_ENTRIES = 1024;
const NON_READY_CACHE_MS = 30_000;
const SUPPORTED_IMAGE_MIME_TYPES = new Set([
'image/png',
'image/jpeg',
@@ -9,37 +11,20 @@ const SUPPORTED_IMAGE_MIME_TYPES = new Set([
'image/webp',
]);
const parseLocalImagePath = (source: string): string => {
let value = source;
if (/^file:\/\//i.test(value)) {
try {
const fileUrl = new URL(value);
if (fileUrl.protocol !== 'file:') return '';
value = fileUrl.host && fileUrl.host !== 'localhost'
? `//${fileUrl.host}${fileUrl.pathname}`
: fileUrl.pathname;
if (/^\/[A-Za-z]:\//.test(value)) value = value.slice(1);
} catch {
return '';
}
}
export type PreparedMarkdownImage =
| { status: 'ready'; path: string; outsideFileGrant?: string; expiresAt?: number }
| { status: 'missing' | 'error' };
const path = value.split(/[?#]/, 1)[0] ?? '';
try {
return decodeURIComponent(path);
} catch {
return path;
}
type PrepareCacheEntry = {
result: Map<string, PreparedMarkdownImage>;
expiresAt: number;
};
const blobToDataUrl = (blob: Blob): Promise<string> => new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => typeof reader.result === 'string'
? resolve(reader.result)
: reject(new Error('Unable to encode image'));
reader.onerror = () => reject(reader.error ?? new Error('Unable to encode image'));
reader.readAsDataURL(blob);
});
const prepareCaches = new WeakMap<RuntimeUrlResolver, Map<string, PrepareCacheEntry>>();
const throwIfAborted = (signal: AbortSignal): void => {
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
};
const hasImageSignature = async (blob: Blob, mimeType: string): Promise<boolean> => {
const bytes = new Uint8Array(await blob.slice(0, 12).arrayBuffer());
@@ -48,9 +33,7 @@ const hasImageSignature = async (blob: Blob, mimeType: string): Promise<boolean>
return bytes[0] === 0x89 && ascii(1, 4) === 'PNG'
&& bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a;
}
if (mimeType === 'image/jpeg') {
return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
}
if (mimeType === 'image/jpeg') return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
if (mimeType === 'image/gif') {
const gif = ascii(0, 6);
return gif === 'GIF87a' || gif === 'GIF89a';
@@ -67,66 +50,123 @@ const validateImageBlob = async (blob: Blob, mimeType: string): Promise<void> =>
const validateDataImage = async (source: string): Promise<void> => {
const match = /^data:(image\/(?:png|jpeg|gif|webp));base64,([\s\S]*)$/i.exec(source);
if (!match?.[1] || match[2] === undefined) throw new Error('Invalid image data URL');
const encoded = match[2];
if (encoded.length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) {
throw new Error('Image is too large');
}
if (match[2].length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) throw new Error('Image is too large');
let binary: string;
try {
binary = atob(encoded);
binary = atob(match[2]);
} catch {
throw new Error('Invalid image data URL');
}
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
await validateImageBlob(new Blob([bytes]), match[1].toLowerCase());
};
export const isLocalMarkdownImageSource = (source: string): boolean => (
!/^(?:https?:)?\/\//i.test(source)
&& !/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)
);
export const prepareLocalMarkdownImages = async ({
sources,
directory,
sessionId,
messageId,
signal,
}: {
sources: readonly string[];
directory: string;
sessionId: string;
messageId: string;
signal: AbortSignal;
}): Promise<Map<string, PreparedMarkdownImage>> => {
const resolver = getRuntimeUrlResolver();
let cache = prepareCaches.get(resolver);
if (!cache) {
cache = new Map();
prepareCaches.set(resolver, cache);
}
const key = `${sessionId}\0${messageId}\0${directory}\0${sources.join('\0')}`;
const cached = cache.get(key);
if (cached && cached.expiresAt > Date.now()) {
cache.delete(key);
cache.set(key, cached);
return cached.result;
}
if (cached) cache.delete(key);
const response = await runtimeFetch(
`/api/openchamber/sessions/${encodeURIComponent(sessionId)}/markdown-image-grants`,
{
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ directory, messageId, sources }),
signal,
},
);
if (!response.ok) throw new Error(`Unable to prepare images (${response.status})`);
const payload = await response.json() as {
results?: Array<{
source?: string;
status?: string;
path?: string;
outsideFileGrant?: string;
expiresAt?: number;
}>;
};
const prepared = new Map<string, PreparedMarkdownImage>();
for (const result of payload.results ?? []) {
if (!result.source) continue;
if (result.status === 'ready' && result.path) {
prepared.set(result.source, {
status: 'ready',
path: result.path,
outsideFileGrant: result.outsideFileGrant,
expiresAt: result.expiresAt,
});
} else if (result.status === 'missing') {
prepared.set(result.source, { status: 'missing' });
} else {
prepared.set(result.source, { status: 'error' });
}
}
for (const source of sources) {
if (!prepared.has(source)) prepared.set(source, { status: 'error' });
}
while (cache.size >= MAX_PREPARE_CACHE_ENTRIES) cache.delete(cache.keys().next().value!);
const allReady = [...prepared.values()].every((value) => value.status === 'ready');
const grantExpiry = Math.min(...[...prepared.values()]
.filter((value): value is Extract<PreparedMarkdownImage, { status: 'ready' }> => value.status === 'ready')
.map((value) => value.expiresAt ?? Number.POSITIVE_INFINITY));
cache.set(key, {
result: prepared,
expiresAt: allReady ? grantExpiry : Date.now() + NON_READY_CACHE_MS,
});
return prepared;
};
export const resolveMarkdownImageSource = async (
source: string,
directory: string,
signal: AbortSignal,
): Promise<string> => {
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
throwIfAborted(signal);
if (/^(?:https?:)?\/\//i.test(source)) return source;
if (/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)) {
await validateDataImage(source);
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
throwIfAborted(signal);
return source;
}
const localPath = parseLocalImagePath(source);
const absolutePath = toAbsoluteFilePath(directory, localPath);
if (!directory || !localPath || !isFilePathWithinDirectory(absolutePath, directory)) {
throw new Error('Image path is outside the active workspace');
}
const statResponse = await runtimeFetch('/api/fs/stat', {
query: { path: absolutePath, directory, optional: 'true' },
signal,
});
if (!statResponse.ok) throw new Error(`Unable to inspect image (${statResponse.status})`);
const stat = await statResponse.json() as { isFile?: boolean; size?: number };
if (!stat.isFile) throw new Error('Image path is not a file');
if (typeof stat.size === 'number' && stat.size > MAX_MARKDOWN_IMAGE_BYTES) {
throw new Error('Image is too large');
}
const response = await runtimeFetch('/api/fs/raw', {
query: { path: absolutePath, directory },
signal,
});
if (!response.ok) throw new Error(`Unable to load image (${response.status})`);
const mimeType = (response.headers.get('content-type') ?? '').split(';', 1)[0]?.toLowerCase() ?? '';
const contentLength = Number(response.headers.get('content-length'));
if (Number.isFinite(contentLength) && contentLength > MAX_MARKDOWN_IMAGE_BYTES) {
throw new Error('Image is too large');
}
const blob = await response.blob();
await validateImageBlob(blob, mimeType);
return blobToDataUrl(blob);
throw new Error('Local image has not been prepared');
};
export const getPreparedMarkdownImageUrl = (
image: Extract<PreparedMarkdownImage, { status: 'ready' }>,
directory: string,
): string => getRuntimeUrlResolver().authenticatedAsset(
'/api/fs/raw',
{
path: image.path,
directory,
allowOutsideWorkspace: image.outsideFileGrant ? 'true' : undefined,
outsideFileGrant: image.outsideFileGrant,
},
);