fix(markdown): correct image gallery rendering (#2894)
This commit is contained in:
ChangeHow
@@ -1,25 +1,81 @@
|
||||
import React from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { Icon } from '@/components/icon/Icon';
|
||||
import { useEffectiveDirectory } from '@/hooks/useEffectiveDirectory';
|
||||
import { useI18n } from '@/lib/i18n';
|
||||
import {
|
||||
acquireRuntimeUrlAuthToken,
|
||||
refreshRuntimeUrlAuthToken,
|
||||
subscribeRuntimeUrlAuthToken,
|
||||
} from '@/lib/runtime-auth';
|
||||
import { getRuntimeApiBaseUrl } from '@/lib/runtime-switch';
|
||||
import type { ToolPopupContent } from './message/types';
|
||||
import {
|
||||
extractMarkdownImageCandidates,
|
||||
MAX_MARKDOWN_IMAGE_COUNT,
|
||||
type MarkdownImageCandidate,
|
||||
} from './markdown/markdownCore';
|
||||
import { resolveMarkdownImageSource } from './markdown/markdownImageAssets';
|
||||
import {
|
||||
getPreparedMarkdownImageUrl,
|
||||
isLocalMarkdownImageSource,
|
||||
prepareLocalMarkdownImages,
|
||||
resolveMarkdownImageSource,
|
||||
type PreparedMarkdownImage,
|
||||
} from './markdown/markdownImageAssets';
|
||||
|
||||
const useAssetAuth = (enabled: boolean): { ready: boolean; nonce: number } => {
|
||||
const [ready, setReady] = React.useState(false);
|
||||
const [nonce, setNonce] = React.useState(0);
|
||||
const apiBaseUrl = getRuntimeApiBaseUrl();
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!enabled) {
|
||||
setReady(false);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
let retryTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const release = acquireRuntimeUrlAuthToken(apiBaseUrl);
|
||||
const unsubscribe = subscribeRuntimeUrlAuthToken(() => {
|
||||
if (!cancelled) setNonce((current) => current + 1);
|
||||
});
|
||||
const refresh = () => {
|
||||
void refreshRuntimeUrlAuthToken(apiBaseUrl)
|
||||
.then(() => {
|
||||
if (!cancelled) setReady(true);
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) retryTimer = setTimeout(refresh, 1000);
|
||||
});
|
||||
};
|
||||
refresh();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (retryTimer) clearTimeout(retryTimer);
|
||||
release();
|
||||
unsubscribe();
|
||||
};
|
||||
}, [apiBaseUrl, enabled]);
|
||||
|
||||
return { ready: !enabled || ready, nonce };
|
||||
};
|
||||
|
||||
const MarkdownImageThumbnail: React.FC<{
|
||||
candidate: MarkdownImageCandidate;
|
||||
preparation?: PreparedMarkdownImage;
|
||||
directory: string;
|
||||
assetAuthReady: boolean;
|
||||
assetAuthNonce: number;
|
||||
onShowPopup?: (content: ToolPopupContent) => void;
|
||||
}> = ({ candidate, directory, onShowPopup }) => {
|
||||
}> = ({ candidate, preparation, directory, assetAuthReady, assetAuthNonce, onShowPopup }) => {
|
||||
const { t } = useI18n();
|
||||
const thumbnailRef = React.useRef<HTMLButtonElement>(null);
|
||||
const [shouldLoad, setShouldLoad] = React.useState(false);
|
||||
const [image, setImage] = React.useState<{
|
||||
url: string;
|
||||
status: 'loading' | 'ready' | 'error';
|
||||
}>({ url: '', status: 'loading' });
|
||||
const [image, setImage] = React.useState<{ url: string; status: 'loading' | 'ready' | 'error' }>({
|
||||
url: '',
|
||||
status: 'loading',
|
||||
});
|
||||
const local = isLocalMarkdownImageSource(candidate.source);
|
||||
|
||||
React.useEffect(() => {
|
||||
const thumbnail = thumbnailRef.current;
|
||||
@@ -28,7 +84,6 @@ const MarkdownImageThumbnail: React.FC<{
|
||||
setShouldLoad(true);
|
||||
return;
|
||||
}
|
||||
|
||||
const observer = new IntersectionObserver((entries) => {
|
||||
if (!entries.some((entry) => entry.isIntersecting)) return;
|
||||
setShouldLoad(true);
|
||||
@@ -39,20 +94,33 @@ const MarkdownImageThumbnail: React.FC<{
|
||||
}, [shouldLoad]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!shouldLoad) return;
|
||||
if (!shouldLoad || (local && !preparation)) return;
|
||||
if (local) {
|
||||
if (preparation?.status !== 'ready') {
|
||||
setImage({ url: '', status: 'error' });
|
||||
return;
|
||||
}
|
||||
if (!assetAuthReady) return;
|
||||
setImage({ url: getPreparedMarkdownImageUrl(preparation, directory), status: 'loading' });
|
||||
return;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
setImage({ url: '', status: 'loading' });
|
||||
void resolveMarkdownImageSource(candidate.source, directory, controller.signal)
|
||||
.then((url) => {
|
||||
if (!controller.signal.aborted) setImage({ url, status: 'loading' });
|
||||
})
|
||||
.catch(() => {
|
||||
if (!controller.signal.aborted) setImage({ url: '', status: 'error' });
|
||||
});
|
||||
void resolveMarkdownImageSource(candidate.source, controller.signal).then((url) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setImage({ url, status: 'loading' });
|
||||
}).catch(() => {
|
||||
if (controller.signal.aborted) return;
|
||||
setImage({ url: '', status: 'error' });
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, [candidate.source, directory, shouldLoad]);
|
||||
}, [assetAuthNonce, assetAuthReady, candidate.source, directory, local, preparation, shouldLoad]);
|
||||
|
||||
const openPreview = React.useCallback(() => {
|
||||
if (image.status === 'error') {
|
||||
toast.error(t('filesView.error.previewUnavailable'));
|
||||
return;
|
||||
}
|
||||
if (image.status !== 'ready' || !onShowPopup) return;
|
||||
onShowPopup({
|
||||
open: true,
|
||||
@@ -61,7 +129,7 @@ const MarkdownImageThumbnail: React.FC<{
|
||||
metadata: { tool: 'markdown-image-preview', filename: candidate.filename },
|
||||
image: { url: image.url, filename: candidate.filename },
|
||||
});
|
||||
}, [candidate.filename, image, onShowPopup]);
|
||||
}, [candidate.filename, image, onShowPopup, t]);
|
||||
|
||||
return (
|
||||
<button
|
||||
@@ -69,7 +137,7 @@ const MarkdownImageThumbnail: React.FC<{
|
||||
type="button"
|
||||
className="w-[100px] shrink-0 text-left outline-none focus-visible:ring-2 focus-visible:ring-[var(--interactive-focus-ring)]"
|
||||
aria-label={candidate.filename}
|
||||
disabled={image.status !== 'ready'}
|
||||
disabled={image.status === 'loading'}
|
||||
onClick={openPreview}
|
||||
data-openchamber-markdown-image-action="true"
|
||||
data-openchamber-markdown-image-source={candidate.source}
|
||||
@@ -107,27 +175,88 @@ const MarkdownImageThumbnail: React.FC<{
|
||||
};
|
||||
|
||||
export const MarkdownImageGallery: React.FC<{
|
||||
sessionId?: string;
|
||||
messageId: string;
|
||||
contents: readonly string[];
|
||||
onShowPopup?: (content: ToolPopupContent) => void;
|
||||
}> = ({ contents, onShowPopup }) => {
|
||||
}> = ({ sessionId, messageId, contents, onShowPopup }) => {
|
||||
const directory = useEffectiveDirectory() ?? '';
|
||||
const galleryRef = React.useRef<HTMLDivElement>(null);
|
||||
const [shouldPrepare, setShouldPrepare] = React.useState(false);
|
||||
const [prepared, setPrepared] = React.useState<Map<string, PreparedMarkdownImage> | null>(null);
|
||||
const [prepareEpoch, setPrepareEpoch] = React.useState(0);
|
||||
const candidates = React.useMemo(
|
||||
() => extractMarkdownImageCandidates(contents, MAX_MARKDOWN_IMAGE_COUNT),
|
||||
[contents],
|
||||
);
|
||||
const localSources = React.useMemo(
|
||||
() => candidates.filter((candidate) => isLocalMarkdownImageSource(candidate.source)).map((candidate) => candidate.source),
|
||||
[candidates],
|
||||
);
|
||||
React.useEffect(() => {
|
||||
if (localSources.length === 0 || shouldPrepare) return;
|
||||
const gallery = galleryRef.current;
|
||||
if (!gallery || typeof IntersectionObserver === 'undefined') {
|
||||
setShouldPrepare(true);
|
||||
return;
|
||||
}
|
||||
const observer = new IntersectionObserver((entries) => {
|
||||
if (!entries.some((entry) => entry.isIntersecting)) return;
|
||||
setShouldPrepare(true);
|
||||
observer.disconnect();
|
||||
}, { rootMargin: '200px' });
|
||||
observer.observe(gallery);
|
||||
return () => observer.disconnect();
|
||||
}, [localSources.length, shouldPrepare]);
|
||||
|
||||
if (candidates.length === 0) return null;
|
||||
React.useEffect(() => {
|
||||
if (!shouldPrepare || !sessionId || localSources.length === 0) return;
|
||||
const controller = new AbortController();
|
||||
void prepareLocalMarkdownImages({
|
||||
sources: localSources,
|
||||
directory,
|
||||
sessionId,
|
||||
messageId,
|
||||
signal: controller.signal,
|
||||
}).then((result) => {
|
||||
if (controller.signal.aborted) return;
|
||||
setPrepared(result);
|
||||
}).catch(() => {
|
||||
if (!controller.signal.aborted) {
|
||||
setPrepared(new Map(localSources.map((source) => [source, { status: 'error' }])));
|
||||
}
|
||||
});
|
||||
return () => controller.abort();
|
||||
}, [directory, localSources, messageId, prepareEpoch, sessionId, shouldPrepare]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const nextExpiry = Math.min(...[...(prepared?.values() ?? [])]
|
||||
.filter((value): value is Extract<PreparedMarkdownImage, { status: 'ready' }> => value.status === 'ready')
|
||||
.map((value) => value.expiresAt ?? Number.POSITIVE_INFINITY));
|
||||
if (!Number.isFinite(nextExpiry)) return;
|
||||
const timer = setTimeout(() => setPrepareEpoch((current) => current + 1), Math.max(0, nextExpiry - Date.now()));
|
||||
return () => clearTimeout(timer);
|
||||
}, [prepared]);
|
||||
|
||||
const visibleCandidates = candidates.filter((candidate) => prepared?.get(candidate.source)?.status !== 'missing');
|
||||
const hasPreparedAssets = [...(prepared?.values() ?? [])].some((value) => value.status === 'ready');
|
||||
const assetAuth = useAssetAuth(hasPreparedAssets);
|
||||
if (visibleCandidates.length === 0) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={galleryRef}
|
||||
className="mt-3 flex max-w-full gap-2 overflow-x-auto pb-1"
|
||||
data-openchamber-markdown-image-gallery="true"
|
||||
>
|
||||
{candidates.map((candidate) => (
|
||||
{visibleCandidates.map((candidate) => (
|
||||
<MarkdownImageThumbnail
|
||||
key={candidate.source}
|
||||
candidate={candidate}
|
||||
preparation={prepared?.get(candidate.source)}
|
||||
directory={directory}
|
||||
assetAuthReady={assetAuth.ready}
|
||||
assetAuthNonce={assetAuth.nonce}
|
||||
onShowPopup={onShowPopup}
|
||||
/>
|
||||
))}
|
||||
|
||||
@@ -19,8 +19,7 @@ import { isDesktopLocalOriginActive, isDesktopShell, isVSCodeRuntime } from '@/l
|
||||
import { isMobileSurfaceRuntime } from '@/lib/runtimeSurface';
|
||||
import { ensureOutsideFileGrantForDesktop } from '@/lib/outsideFileGrants';
|
||||
import { getDirectoryForFilePath, isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils';
|
||||
import { getMarkdownImageFilename, renderMarkdownBlocks, renderMarkdownSync } from './markdown/markdownCore';
|
||||
import { resolveMarkdownImageSource } from './markdown/markdownImageAssets';
|
||||
import { renderMarkdownBlocks, renderMarkdownSync, type MarkdownImageMode } from './markdown/markdownCore';
|
||||
import { ensureMarkdownShikiTheme } from './markdown/markdownTheme';
|
||||
import { getMarkdownSyntaxVars } from './markdown/markdownSyntaxVars';
|
||||
import {
|
||||
@@ -109,59 +108,6 @@ const useExternalLinkInteractions = ({
|
||||
}, [containerRef, enabled]);
|
||||
};
|
||||
|
||||
const useMarkdownImageLinkInteractions = ({
|
||||
containerRef,
|
||||
directory,
|
||||
enabled,
|
||||
onShowPopup,
|
||||
}: {
|
||||
containerRef: React.RefObject<HTMLDivElement | null>;
|
||||
directory: string;
|
||||
enabled: boolean;
|
||||
onShowPopup?: (content: ToolPopupContent) => void;
|
||||
}) => {
|
||||
React.useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
if (!enabled || !container || !onShowPopup) return;
|
||||
|
||||
const controller = new AbortController();
|
||||
const handleClick = (event: MouseEvent) => {
|
||||
if (event.defaultPrevented || event.button !== 0) return;
|
||||
const target = event.target;
|
||||
if (!(target instanceof Element)) return;
|
||||
|
||||
const link = target.closest<HTMLAnchorElement>('[data-openchamber-markdown-image-link="true"]');
|
||||
if (!link || !container.contains(link)) return;
|
||||
|
||||
const source = link.getAttribute('data-openchamber-markdown-image-source') ?? '';
|
||||
const filename = link.getAttribute('data-openchamber-markdown-image-filename')
|
||||
|| getMarkdownImageFilename(source, '');
|
||||
if (!source || !filename) return;
|
||||
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
void resolveMarkdownImageSource(source, directory, controller.signal)
|
||||
.then((url) => {
|
||||
if (controller.signal.aborted || !link.isConnected) return;
|
||||
onShowPopup({
|
||||
open: true,
|
||||
title: filename,
|
||||
content: '',
|
||||
metadata: { tool: 'markdown-image-preview', filename },
|
||||
image: { url, filename },
|
||||
});
|
||||
})
|
||||
.catch(() => undefined);
|
||||
};
|
||||
|
||||
container.addEventListener('click', handleClick);
|
||||
return () => {
|
||||
controller.abort();
|
||||
container.removeEventListener('click', handleClick);
|
||||
};
|
||||
}, [containerRef, directory, enabled, onShowPopup]);
|
||||
};
|
||||
|
||||
const DEFAULT_MERMAID_CONTROLS: MermaidControlOptions = {
|
||||
download: true,
|
||||
copy: true,
|
||||
@@ -195,7 +141,6 @@ interface MarkdownRendererProps {
|
||||
variant?: MarkdownVariant;
|
||||
onShowPopup?: (content: ToolPopupContent) => void;
|
||||
enableFileReferences?: boolean;
|
||||
enableLocalImages?: boolean;
|
||||
}
|
||||
|
||||
const FILE_LINK_SELECTOR = '[data-openchamber-file-link="true"]';
|
||||
@@ -557,10 +502,6 @@ const useFileReferenceInteractions = ({
|
||||
let linkedCount = 0;
|
||||
|
||||
for (const candidate of Array.from(candidates)) {
|
||||
if (candidate.matches('[data-openchamber-markdown-image-link="true"]')) {
|
||||
clearFileLinkAttributes(candidate);
|
||||
continue;
|
||||
}
|
||||
const rawCandidate = extractPathCandidateFromElement(candidate);
|
||||
const resolved = getResolvedReference(rawCandidate, effectiveDirectory);
|
||||
clearFileLinkAttributes(candidate);
|
||||
@@ -895,7 +836,7 @@ const useMorphdomMarkdown = ({
|
||||
text,
|
||||
streaming,
|
||||
cacheKey,
|
||||
deferImages = false,
|
||||
imageMode = 'inline',
|
||||
syntaxVars,
|
||||
ctx,
|
||||
}: {
|
||||
@@ -903,7 +844,7 @@ const useMorphdomMarkdown = ({
|
||||
text: string;
|
||||
streaming: boolean;
|
||||
cacheKey: string;
|
||||
deferImages?: boolean;
|
||||
imageMode?: MarkdownImageMode;
|
||||
syntaxVars: Record<string, string>;
|
||||
ctx: DecorateContext;
|
||||
}) => {
|
||||
@@ -942,7 +883,7 @@ const useMorphdomMarkdown = ({
|
||||
// `display:contents` keeps margin-collapsing/spacing identical to a flat
|
||||
// HTML body — the wrapper exists only for per-block reconciliation.
|
||||
block.style.display = 'contents';
|
||||
block.innerHTML = renderMarkdownSync(text, deferImages);
|
||||
block.innerHTML = renderMarkdownSync(text, imageMode);
|
||||
// Decorate synchronously too: wrap code blocks in their framed card,
|
||||
// mark inline code, build table controls, etc. The async pass re-decorates
|
||||
// its own DOM before morphing, so without this the first paint shows bare
|
||||
@@ -954,7 +895,7 @@ const useMorphdomMarkdown = ({
|
||||
refreshMermaidViewers();
|
||||
}
|
||||
}
|
||||
}, [containerRef, text, deferImages, ctx, refreshMermaidViewers]);
|
||||
}, [containerRef, text, imageMode, ctx, refreshMermaidViewers]);
|
||||
|
||||
React.useEffect(() => () => {
|
||||
mermaidViewerRef.current?.cleanup();
|
||||
@@ -967,7 +908,7 @@ const useMorphdomMarkdown = ({
|
||||
const target = container.querySelector<HTMLElement>('[data-markdown-content]') ?? container;
|
||||
let active = true;
|
||||
|
||||
void renderMarkdownBlocks(text, streaming, cacheKey, deferImages).then((blocks) => {
|
||||
void renderMarkdownBlocks(text, streaming, cacheKey, imageMode).then((blocks) => {
|
||||
if (!active) return;
|
||||
const existing = Array.from(target.children) as HTMLElement[];
|
||||
|
||||
@@ -1018,7 +959,7 @@ const useMorphdomMarkdown = ({
|
||||
return () => {
|
||||
active = false;
|
||||
};
|
||||
}, [containerRef, text, streaming, cacheKey, deferImages, ctx, refreshMermaidViewers]);
|
||||
}, [containerRef, text, streaming, cacheKey, imageMode, ctx, refreshMermaidViewers]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
@@ -1065,7 +1006,6 @@ const MarkdownRendererImpl: React.FC<MarkdownRendererProps> = ({
|
||||
variant = 'assistant',
|
||||
onShowPopup,
|
||||
enableFileReferences = true,
|
||||
enableLocalImages = false,
|
||||
}) => {
|
||||
streamPerfCount('ui.markdown_renderer.render');
|
||||
if (isStreaming) streamPerfCount('ui.markdown_renderer.render.streaming');
|
||||
@@ -1097,12 +1037,6 @@ const MarkdownRendererImpl: React.FC<MarkdownRendererProps> = ({
|
||||
enabled: enableFileReferences && !isStreaming,
|
||||
});
|
||||
useExternalLinkInteractions({ containerRef });
|
||||
useMarkdownImageLinkInteractions({
|
||||
containerRef,
|
||||
directory: effectiveDirectory,
|
||||
enabled: enableLocalImages && variant === 'assistant' && !isStreaming,
|
||||
onShowPopup,
|
||||
});
|
||||
|
||||
const syntaxVars = React.useMemo(() => getMarkdownSyntaxVars(currentTheme), [currentTheme]);
|
||||
const ctx = useDecorateContext(currentTheme, live, effectiveDirectory ? handlePreviewLoopback : undefined, DEFAULT_MERMAID_CONTROLS);
|
||||
@@ -1113,17 +1047,13 @@ const MarkdownRendererImpl: React.FC<MarkdownRendererProps> = ({
|
||||
text: content,
|
||||
streaming: live,
|
||||
cacheKey,
|
||||
deferImages: enableLocalImages && variant === 'assistant' && !isStreaming,
|
||||
imageMode: variant === 'assistant' ? 'label' : 'inline',
|
||||
syntaxVars,
|
||||
ctx,
|
||||
});
|
||||
|
||||
const markdownContent = (
|
||||
<div
|
||||
className={cn('break-words w-full min-w-0', className)}
|
||||
ref={containerRef}
|
||||
data-openchamber-finalized-assistant-images={enableLocalImages && variant === 'assistant' && !isStreaming ? 'true' : undefined}
|
||||
>
|
||||
<div className={cn('break-words w-full min-w-0', className)} ref={containerRef}>
|
||||
<div className={markdownContentClassName(variant)} data-markdown-content />
|
||||
</div>
|
||||
);
|
||||
@@ -1150,7 +1080,6 @@ export const MarkdownRenderer = React.memo(MarkdownRendererImpl, (prev, next) =>
|
||||
&& prev.messageId === next.messageId
|
||||
&& prev.onShowPopup === next.onShowPopup
|
||||
&& prev.enableFileReferences === next.enableFileReferences
|
||||
&& prev.enableLocalImages === next.enableLocalImages
|
||||
&& prev.part?.id === next.part?.id;
|
||||
});
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ const ICONS = {
|
||||
zoomOut: spriteIcon('subtract'),
|
||||
fit: spriteIcon('refresh'),
|
||||
textWrap: spriteIcon('text-wrap'),
|
||||
image: spriteIcon('file-image'),
|
||||
} as const;
|
||||
|
||||
const ICON_BTN_CLASS =
|
||||
@@ -66,6 +67,18 @@ const setIconHtml = (el: Element, html: string): void => {
|
||||
el.innerHTML = html;
|
||||
};
|
||||
|
||||
const decorateImageLabels = (root: HTMLElement): void => {
|
||||
for (const label of Array.from(root.querySelectorAll<HTMLElement>('[data-openchamber-markdown-image-label="true"]'))) {
|
||||
if (label.querySelector('[data-openchamber-markdown-image-label-icon]')) continue;
|
||||
const icon = document.createElement('span');
|
||||
icon.className = 'inline-flex shrink-0';
|
||||
icon.setAttribute('aria-hidden', 'true');
|
||||
icon.setAttribute('data-openchamber-markdown-image-label-icon', 'true');
|
||||
setIconHtml(icon, ICONS.image);
|
||||
label.prepend(icon);
|
||||
}
|
||||
};
|
||||
|
||||
const makeIconButton = (icon: keyof typeof ICONS, title: string, slot: string): HTMLButtonElement => {
|
||||
const button = document.createElement('button');
|
||||
button.type = 'button';
|
||||
@@ -487,6 +500,7 @@ const decorateLinks = (root: HTMLElement, ctx: DecorateContext): void => {
|
||||
|
||||
/** Run all idempotent DOM decoration passes over freshly-rendered markdown. */
|
||||
export const decorateMarkdown = (root: HTMLElement, ctx: DecorateContext): void => {
|
||||
decorateImageLabels(root);
|
||||
decorateInlineCode(root);
|
||||
decorateMermaid(root, ctx);
|
||||
decorateCodeBlocks(root, ctx);
|
||||
|
||||
@@ -13,7 +13,11 @@ mock.module('./markdown-worker', () => ({
|
||||
|
||||
import { escapeRawMarkdownHtml, isLocalFileUrl, MARKDOWN_FORBIDDEN_TAGS } from './markdownSecurity';
|
||||
|
||||
const { extractMarkdownImageCandidates, renderMarkdownSync } = await import('./markdownCore');
|
||||
const {
|
||||
__markdownImageCandidateCacheForTests,
|
||||
extractMarkdownImageCandidates,
|
||||
renderMarkdownSync,
|
||||
} = await import('./markdownCore');
|
||||
const { resolveMarkdownImageSource } = await import('./markdownImageAssets');
|
||||
|
||||
describe('markdown sanitization', () => {
|
||||
@@ -39,45 +43,31 @@ describe('markdown sanitization', () => {
|
||||
});
|
||||
|
||||
describe('Markdown images', () => {
|
||||
test('keeps local image links in text and emits inert image placeholders', () => {
|
||||
test('renders assistant images as icon-ready text without loading the source', () => {
|
||||
const html = renderMarkdownSync([
|
||||
'[linked image](packages/vscode/extension.jpg)',
|
||||
'',
|
||||
].join('\n\n'), true);
|
||||
].join('\n\n'), 'label');
|
||||
|
||||
expect(html).toContain('data-openchamber-markdown-image-link="true"');
|
||||
expect(html.match(/data-openchamber-markdown-image-source="packages\/vscode\/extension.jpg"/g)).toHaveLength(1);
|
||||
expect(html).toContain('data-openchamber-markdown-image-placeholder="true"');
|
||||
expect(html).toContain('image syntax');
|
||||
expect(html).not.toContain('src="packages/vscode/extension.jpg"');
|
||||
expect(html).not.toContain('data-openchamber-markdown-image-state');
|
||||
expect(html).toContain('data-openchamber-markdown-image-label="true"');
|
||||
expect(html).toContain('extension.jpg');
|
||||
expect(html).not.toContain('image syntax');
|
||||
expect(html).not.toContain('<img');
|
||||
expect(html.match(/<a /g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('keeps HTTP links as links and defers remote image tokens to finalized rendering', () => {
|
||||
test('keeps non-chat Markdown images inline', () => {
|
||||
const html = renderMarkdownSync([
|
||||
'[remote link](https://example.test/image.png)',
|
||||
'',
|
||||
].join('\n\n'), true);
|
||||
].join('\n\n'));
|
||||
|
||||
expect(html).toContain('<a href="https://example.test/image.png"');
|
||||
expect(html).not.toContain('<img');
|
||||
expect(html).toContain('data-openchamber-markdown-image-placeholder="true"');
|
||||
expect(html).toContain('remote image');
|
||||
expect(html).toContain('<img src="https://example.test/image.png" alt="remote image">');
|
||||
expect(html).not.toContain('data-openchamber-markdown-image-label');
|
||||
});
|
||||
|
||||
test('preserves file URLs inertly and never activates unknown schemes', () => {
|
||||
const html = renderMarkdownSync([
|
||||
'',
|
||||
')',
|
||||
].join('\n\n'), true);
|
||||
|
||||
expect(html).toContain('role="img"');
|
||||
expect(html).not.toContain('src="file:');
|
||||
expect(html).not.toContain('src="javascript:');
|
||||
});
|
||||
|
||||
test('collects a single ordered gallery across mixed Markdown and ignores code', () => {
|
||||
test('collects image syntax across mixed Markdown and ignores links and code', () => {
|
||||
const candidates = extractMarkdownImageCandidates([
|
||||
[
|
||||
'Before [local link](screens/first%20view.png) and ``.',
|
||||
@@ -99,6 +89,10 @@ describe('Markdown images', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test('does not add an ordinary local image link to the gallery', () => {
|
||||
expect(extractMarkdownImageCandidates(['[download](screens/image.png)'])).toEqual([]);
|
||||
});
|
||||
|
||||
test('limits one finalized message gallery to twelve unique candidates', () => {
|
||||
const markdown = Array.from({ length: 14 }, (_, index) => ``).join('\n');
|
||||
|
||||
@@ -108,12 +102,69 @@ describe('Markdown images', () => {
|
||||
expect(candidates.at(-1)?.source).toBe('screens/11.png');
|
||||
});
|
||||
|
||||
test('reuses extracted candidates across virtualized remounts without changing gallery behavior', () => {
|
||||
__markdownImageCandidateCacheForTests.reset();
|
||||
const contents = Array.from({ length: 20 }, (_, index) => ``);
|
||||
|
||||
expect(extractMarkdownImageCandidates(contents)).toHaveLength(12);
|
||||
expect(__markdownImageCandidateCacheForTests.stats().scans).toBe(12);
|
||||
|
||||
for (let round = 0; round < 1000; round += 1) {
|
||||
expect(extractMarkdownImageCandidates(contents)).toHaveLength(12);
|
||||
}
|
||||
|
||||
const stats = __markdownImageCandidateCacheForTests.stats();
|
||||
expect(stats.entries).toBe(12);
|
||||
expect(stats.scans).toBe(12);
|
||||
});
|
||||
|
||||
test('scans one thousand independent messages once across virtualized remounts', () => {
|
||||
__markdownImageCandidateCacheForTests.reset();
|
||||
const messages = Array.from(
|
||||
{ length: 1000 },
|
||||
(_, index) => ``,
|
||||
);
|
||||
|
||||
for (const message of messages) extractMarkdownImageCandidates([message]);
|
||||
for (const message of messages) extractMarkdownImageCandidates([message]);
|
||||
|
||||
const stats = __markdownImageCandidateCacheForTests.stats();
|
||||
expect(stats.entries).toBe(1000);
|
||||
expect(stats.scans).toBe(1000);
|
||||
});
|
||||
|
||||
test('gives embedded images without alt text a stable filename', () => {
|
||||
const source = 'data:image/png;base64,AAAA';
|
||||
|
||||
expect(extractMarkdownImageCandidates([``])).toEqual([
|
||||
{ source, filename: 'image.png' },
|
||||
]);
|
||||
expect(renderMarkdownSync(``, 'label')).toContain('image.png');
|
||||
});
|
||||
|
||||
test('bounds cached candidate entries and bytes, and skips oversized individual content', () => {
|
||||
__markdownImageCandidateCacheForTests.reset();
|
||||
for (let index = 0; index < 1025; index += 1) {
|
||||
extractMarkdownImageCandidates([``]);
|
||||
}
|
||||
const boundedStats = __markdownImageCandidateCacheForTests.stats();
|
||||
expect(boundedStats.entries).toBe(1024);
|
||||
expect(boundedStats.bytes <= 2 * 1024 * 1024).toBe(true);
|
||||
|
||||
__markdownImageCandidateCacheForTests.reset();
|
||||
const oversized = `\n${'x'.repeat(64 * 1024)}`;
|
||||
|
||||
extractMarkdownImageCandidates([oversized]);
|
||||
extractMarkdownImageCandidates([oversized]);
|
||||
expect(__markdownImageCandidateCacheForTests.stats()).toEqual({ entries: 0, bytes: 0, scans: 2 });
|
||||
});
|
||||
|
||||
test('validates embedded image bytes against the declared MIME type', async () => {
|
||||
const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==';
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, '', signal)).toBe(`data:image/png;base64,${png}`);
|
||||
await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, '', signal).then(
|
||||
expect(await resolveMarkdownImageSource(`data:image/png;base64,${png}`, signal)).toBe(`data:image/png;base64,${png}`);
|
||||
await resolveMarkdownImageSource(`data:image/jpeg;base64,${png}`, signal).then(
|
||||
() => { throw new Error('Expected mismatched image data to fail'); },
|
||||
(error: unknown) => expect((error as Error).message).toBe('Unsupported image data'),
|
||||
);
|
||||
@@ -123,7 +174,7 @@ describe('Markdown images', () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await resolveMarkdownImageSource('https://example.test/image.png', '', controller.signal).then(
|
||||
await resolveMarkdownImageSource('https://example.test/image.png', controller.signal).then(
|
||||
() => { throw new Error('Expected an aborted image load to fail'); },
|
||||
(error: unknown) => expect((error as Error).name).toBe('AbortError'),
|
||||
);
|
||||
@@ -133,6 +184,6 @@ describe('Markdown images', () => {
|
||||
const html = renderMarkdownSync('');
|
||||
|
||||
expect(html).toContain('<img src="https://example.test/image.png"');
|
||||
expect(html).not.toContain('data-openchamber-markdown-image-placeholder');
|
||||
expect(html).not.toContain('data-openchamber-markdown-image');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,8 +19,23 @@ export interface MarkdownImageCandidate {
|
||||
filename: string;
|
||||
}
|
||||
|
||||
export type MarkdownImageMode = 'inline' | 'label';
|
||||
|
||||
export const MAX_MARKDOWN_IMAGE_COUNT = 12;
|
||||
|
||||
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRIES = 1024;
|
||||
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_BYTES = 2 * 1024 * 1024;
|
||||
const MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRY_BYTES = 64 * 1024;
|
||||
|
||||
type MarkdownImageCandidateCacheEntry = {
|
||||
candidates: MarkdownImageCandidate[];
|
||||
bytes: number;
|
||||
};
|
||||
|
||||
const markdownImageCandidateCache = new Map<string, MarkdownImageCandidateCacheEntry>();
|
||||
let markdownImageCandidateCacheBytes = 0;
|
||||
let markdownImageCandidateScanCount = 0;
|
||||
|
||||
const isLocalMarkdownImageSource = (source: string): boolean => {
|
||||
if (/^\/\//.test(source) || !LOCAL_IMAGE_EXTENSION_RE.test(source)) return false;
|
||||
return WINDOWS_ABSOLUTE_PATH_RE.test(source)
|
||||
@@ -34,8 +49,11 @@ const isSupportedMarkdownImageSource = (source: string): boolean => (
|
||||
|| isLocalMarkdownImageSource(source)
|
||||
);
|
||||
|
||||
export const getMarkdownImageFilename = (source: string, fallback: string): string => {
|
||||
if (/^data:/i.test(source)) return fallback.trim();
|
||||
const getMarkdownImageFilename = (source: string, fallback: string): string => {
|
||||
if (/^data:image\/(png|jpeg|gif|webp)/i.test(source)) {
|
||||
const extension = /^data:image\/([^;,]+)/i.exec(source)?.[1]?.replace('jpeg', 'jpg') ?? 'png';
|
||||
return fallback.trim() || `image.${extension}`;
|
||||
}
|
||||
|
||||
const path = source.split(/[?#]/, 1)[0]?.replace(/\\/g, '/') ?? '';
|
||||
const encodedName = path.split('/').filter(Boolean).at(-1) ?? '';
|
||||
@@ -47,6 +65,87 @@ export const getMarkdownImageFilename = (source: string, fallback: string): stri
|
||||
}
|
||||
};
|
||||
|
||||
const estimateMarkdownImageCandidateCacheEntryBytes = (
|
||||
markdown: string,
|
||||
candidates: readonly MarkdownImageCandidate[],
|
||||
): number => (
|
||||
(markdown.length + candidates.reduce((total, candidate) => total + candidate.source.length + candidate.filename.length, 0)) * 2
|
||||
);
|
||||
|
||||
const scanMarkdownImageCandidates = (markdown: string): MarkdownImageCandidate[] => {
|
||||
markdownImageCandidateScanCount += 1;
|
||||
const candidates: MarkdownImageCandidate[] = [];
|
||||
const seen = new Set<string>();
|
||||
const tokens = marked.lexer(markdown);
|
||||
marked.walkTokens(tokens, (token) => {
|
||||
if (token.type !== 'image') return;
|
||||
|
||||
const source = token.href ?? '';
|
||||
if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return;
|
||||
const fallback = typeof token.text === 'string' ? token.text : '';
|
||||
const filename = getMarkdownImageFilename(source, fallback);
|
||||
if (!filename) return;
|
||||
|
||||
seen.add(source);
|
||||
candidates.push({ source, filename });
|
||||
});
|
||||
return candidates;
|
||||
};
|
||||
|
||||
const getMarkdownImageCandidates = (markdown: string): MarkdownImageCandidate[] => {
|
||||
const cached = markdownImageCandidateCache.get(markdown);
|
||||
if (cached) {
|
||||
markdownImageCandidateCache.delete(markdown);
|
||||
markdownImageCandidateCache.set(markdown, cached);
|
||||
return cached.candidates;
|
||||
}
|
||||
|
||||
const candidates = scanMarkdownImageCandidates(markdown);
|
||||
const bytes = estimateMarkdownImageCandidateCacheEntryBytes(markdown, candidates);
|
||||
if (bytes > MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRY_BYTES) return candidates;
|
||||
|
||||
while (
|
||||
markdownImageCandidateCache.size >= MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_ENTRIES
|
||||
|| markdownImageCandidateCacheBytes + bytes > MARKDOWN_IMAGE_CANDIDATE_CACHE_MAX_BYTES
|
||||
) {
|
||||
const oldest = markdownImageCandidateCache.entries().next().value;
|
||||
if (!oldest) break;
|
||||
markdownImageCandidateCache.delete(oldest[0]);
|
||||
markdownImageCandidateCacheBytes -= oldest[1].bytes;
|
||||
}
|
||||
markdownImageCandidateCache.set(markdown, { candidates, bytes });
|
||||
markdownImageCandidateCacheBytes += bytes;
|
||||
return candidates;
|
||||
};
|
||||
|
||||
/** @internal Test-only cache instrumentation for deterministic regression tests. */
|
||||
export const __markdownImageCandidateCacheForTests = {
|
||||
reset: (): void => {
|
||||
markdownImageCandidateCache.clear();
|
||||
markdownImageCandidateCacheBytes = 0;
|
||||
markdownImageCandidateScanCount = 0;
|
||||
},
|
||||
stats: () => ({
|
||||
entries: markdownImageCandidateCache.size,
|
||||
bytes: markdownImageCandidateCacheBytes,
|
||||
scans: markdownImageCandidateScanCount,
|
||||
}),
|
||||
};
|
||||
|
||||
const renderMarkdownImageLabel = ({
|
||||
href,
|
||||
title,
|
||||
text,
|
||||
}: {
|
||||
href: string;
|
||||
title?: string | null;
|
||||
text: string;
|
||||
}): string => {
|
||||
const label = getMarkdownImageFilename(href ?? '', text);
|
||||
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
|
||||
return `<span${titleAttr} class="inline-flex items-center gap-1 align-text-bottom text-muted-foreground" data-openchamber-markdown-image-label="true">${escapeAttr(label)}</span>`;
|
||||
};
|
||||
|
||||
export const extractMarkdownImageCandidates = (
|
||||
markdownTexts: readonly string[],
|
||||
limit = MAX_MARKDOWN_IMAGE_COUNT,
|
||||
@@ -58,47 +157,17 @@ export const extractMarkdownImageCandidates = (
|
||||
|
||||
for (const markdown of markdownTexts) {
|
||||
if (!markdown || candidates.length >= limit) continue;
|
||||
const tokens = marked.lexer(markdown);
|
||||
marked.walkTokens(tokens, (token) => {
|
||||
if (candidates.length >= limit) return;
|
||||
|
||||
if (token.type !== 'image' && token.type !== 'link') return;
|
||||
if (token.type === 'link' && !isLocalMarkdownImageSource(token.href ?? '')) return;
|
||||
|
||||
const source = token.href ?? '';
|
||||
if (!source || !isSupportedMarkdownImageSource(source) || seen.has(source)) return;
|
||||
const fallback = typeof token.text === 'string' ? token.text : '';
|
||||
const filename = getMarkdownImageFilename(source, fallback);
|
||||
if (!filename) return;
|
||||
|
||||
seen.add(source);
|
||||
candidates.push({ source, filename });
|
||||
});
|
||||
for (const candidate of getMarkdownImageCandidates(markdown)) {
|
||||
if (candidates.length >= limit) break;
|
||||
if (seen.has(candidate.source)) continue;
|
||||
seen.add(candidate.source);
|
||||
candidates.push({ ...candidate });
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
};
|
||||
|
||||
const renderMarkdownImage = ({
|
||||
href,
|
||||
title,
|
||||
text,
|
||||
}: {
|
||||
href: string;
|
||||
title?: string | null;
|
||||
text: string;
|
||||
}): string => {
|
||||
const source = href ?? '';
|
||||
const alt = escapeAttr(text ?? '');
|
||||
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
|
||||
const supported = isSupportedMarkdownImageSource(source);
|
||||
if (!supported) {
|
||||
return `<span role="img" aria-label="${alt}"${titleAttr}>${alt}</span>`;
|
||||
}
|
||||
|
||||
return `<span role="img" aria-label="${alt}"${titleAttr} data-openchamber-markdown-image-placeholder="true">${alt}</span>`;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Streaming block segmentation (port of OpenCode's markdown-stream)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -251,7 +320,7 @@ const blockMathExtension = {
|
||||
},
|
||||
};
|
||||
|
||||
const createParser = (deferImages: boolean) => new Marked().use({
|
||||
const createParser = (imageMode: MarkdownImageMode) => new Marked().use({
|
||||
gfm: true,
|
||||
breaks: false,
|
||||
extensions: [inlineMathExtension, blockMathExtension],
|
||||
@@ -264,11 +333,6 @@ const createParser = (deferImages: boolean) => new Marked().use({
|
||||
},
|
||||
link({ href, title, text }) {
|
||||
const target = href ?? '';
|
||||
if (deferImages && isLocalMarkdownImageSource(target)) {
|
||||
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
|
||||
const filename = getMarkdownImageFilename(target, '');
|
||||
return `<a href="${escapeAttr(target)}"${titleAttr} class="external-link" target="_blank" rel="noopener noreferrer" data-openchamber-markdown-image-link="true" data-openchamber-markdown-image-source="${escapeAttr(target)}" data-openchamber-markdown-image-filename="${escapeAttr(filename)}">${text}</a>`;
|
||||
}
|
||||
const agentName = parseAgentHref(target);
|
||||
if (agentName) {
|
||||
return `<a href="${escapeAttr(buildAgentMentionUrl(agentName))}" data-openchamber-agent-mention="true" class="text-primary hover:underline" target="_blank" rel="noopener noreferrer">${text}</a>`;
|
||||
@@ -280,12 +344,12 @@ const createParser = (deferImages: boolean) => new Marked().use({
|
||||
const titleAttr = title ? ` title="${escapeAttr(title)}"` : '';
|
||||
return `<a href="${escapeAttr(target)}"${titleAttr} class="external-link" target="_blank" rel="noopener noreferrer">${text}</a>`;
|
||||
},
|
||||
...(deferImages ? { image: renderMarkdownImage } : {}),
|
||||
...(imageMode === 'label' ? { image: renderMarkdownImageLabel } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
const parser = createParser(false);
|
||||
const imageParser = createParser(true);
|
||||
const inlineImageParser = createParser('inline');
|
||||
const imageLabelParser = createParser('label');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Math (KaTeX) — post-process the parsed HTML, skipping code/pre/kbd content
|
||||
@@ -443,8 +507,9 @@ const touch = (key: string, entry: { hash: string; html: string }): void => {
|
||||
if (oldest) htmlCache.delete(oldest);
|
||||
};
|
||||
|
||||
const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise<string> => {
|
||||
const parsed = await Promise.resolve((deferImages ? imageParser : parser).parse(block.src));
|
||||
const parseBlock = async (block: MarkdownBlock, imageMode: MarkdownImageMode): Promise<string> => {
|
||||
const parser = imageMode === 'label' ? imageLabelParser : inlineImageParser;
|
||||
const parsed = await Promise.resolve(parser.parse(block.src));
|
||||
const withMath = renderMathExpressions(parsed);
|
||||
const highlighted = block.highlight ? await highlightCodeBlocks(withMath) : withMath;
|
||||
return sanitize(highlighted);
|
||||
@@ -459,9 +524,10 @@ const parseBlock = async (block: MarkdownBlock, deferImages: boolean): Promise<s
|
||||
* is synchronous (marked is not configured `async`), so this never blocks on a
|
||||
* worker round-trip.
|
||||
*/
|
||||
export const renderMarkdownSync = (text: string, deferImages = false): string => {
|
||||
export const renderMarkdownSync = (text: string, imageMode: MarkdownImageMode = 'inline'): string => {
|
||||
if (!text) return '';
|
||||
const parsed = (deferImages ? imageParser : parser).parse(text) as string;
|
||||
const parser = imageMode === 'label' ? imageLabelParser : inlineImageParser;
|
||||
const parsed = parser.parse(text) as string;
|
||||
const withMath = renderMathExpressions(parsed);
|
||||
return sanitize(withMath);
|
||||
};
|
||||
@@ -484,7 +550,7 @@ export const renderMarkdownBlocks = async (
|
||||
text: string,
|
||||
streaming: boolean,
|
||||
cacheKey: string,
|
||||
deferImages = false,
|
||||
imageMode: MarkdownImageMode = 'inline',
|
||||
): Promise<RenderedBlock[]> => {
|
||||
if (!text) return [];
|
||||
|
||||
@@ -492,14 +558,14 @@ export const renderMarkdownBlocks = async (
|
||||
return Promise.all(
|
||||
blocks.map(async (block, index) => {
|
||||
const contentHash = hash(block.raw);
|
||||
const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${deferImages ? 1 : 0}`;
|
||||
const key = `${cacheKey}:${index}:${block.mode}:${deferImages ? 1 : 0}`;
|
||||
const id = `${contentHash}:${block.mode}:${block.highlight ? 1 : 0}:${imageMode}`;
|
||||
const key = `${cacheKey}:${index}:${block.mode}:${imageMode}`;
|
||||
const cached = htmlCache.get(key);
|
||||
if (cached && cached.hash === contentHash) {
|
||||
touch(key, cached);
|
||||
return { id, html: cached.html };
|
||||
}
|
||||
const html = await parseBlock(block, deferImages);
|
||||
const html = await parseBlock(block, imageMode);
|
||||
touch(key, { hash: contentHash, html });
|
||||
return { id, html };
|
||||
}),
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { describe, expect, mock, test } from 'bun:test';
|
||||
|
||||
let requestCount = 0;
|
||||
const runtimeFetch = mock(async (_path: string, init?: RequestInit) => {
|
||||
requestCount += 1;
|
||||
const body = JSON.parse(String(init?.body)) as { sources: string[] };
|
||||
return new Response(JSON.stringify({
|
||||
results: body.sources.map((source) => ({ source, status: 'ready', path: `/repo/${source}` })),
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
});
|
||||
const resolver = {
|
||||
api: () => '',
|
||||
authenticatedAsset: (path: string, query: Record<string, string | undefined>) => {
|
||||
const params = new URLSearchParams(Object.entries(query).filter((entry): entry is [string, string] => Boolean(entry[1])));
|
||||
return `${path}?${params}`;
|
||||
},
|
||||
};
|
||||
|
||||
mock.module('@/lib/runtime-fetch', () => ({ runtimeFetch }));
|
||||
mock.module('@/lib/runtime-url', () => ({ getRuntimeUrlResolver: () => resolver }));
|
||||
|
||||
const { getPreparedMarkdownImageUrl, prepareLocalMarkdownImages } = await import('./markdownImageAssets');
|
||||
|
||||
describe('Markdown image asset preparation', () => {
|
||||
test('prepares many images in one message-level request', async () => {
|
||||
requestCount = 0;
|
||||
const sources = Array.from({ length: 12 }, (_, index) => `${index}.png`);
|
||||
|
||||
const result = await prepareLocalMarkdownImages({
|
||||
sources,
|
||||
directory: '/repo',
|
||||
sessionId: 'ses_batch',
|
||||
messageId: 'msg_batch',
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
|
||||
expect(result.size).toBe(12);
|
||||
expect(requestCount).toBe(1);
|
||||
});
|
||||
|
||||
test('reuses preparation for one thousand messages after virtualized remounts', async () => {
|
||||
requestCount = 0;
|
||||
const requests = Array.from({ length: 1000 }, (_, index) => ({
|
||||
sources: [`${index}.png`],
|
||||
directory: '/repo',
|
||||
sessionId: 'ses_long',
|
||||
messageId: `msg_${index}`,
|
||||
signal: new AbortController().signal,
|
||||
}));
|
||||
|
||||
for (const request of requests) await prepareLocalMarkdownImages(request);
|
||||
for (const request of requests) await prepareLocalMarkdownImages(request);
|
||||
|
||||
expect(requestCount).toBe(1000);
|
||||
});
|
||||
|
||||
test('reuses the existing authenticated raw-file asset URL', () => {
|
||||
const url = getPreparedMarkdownImageUrl({
|
||||
status: 'ready',
|
||||
path: '/tmp/opencode/image.png',
|
||||
outsideFileGrant: 'grant-1',
|
||||
}, '/repo');
|
||||
|
||||
expect(url).toContain('/api/fs/raw?');
|
||||
expect(url).toContain('path=%2Ftmp%2Fopencode%2Fimage.png');
|
||||
expect(url).toContain('outsideFileGrant=grant-1');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,9 @@
|
||||
import { isFilePathWithinDirectory, toAbsoluteFilePath } from '@/lib/path-utils';
|
||||
import { runtimeFetch } from '@/lib/runtime-fetch';
|
||||
import { getRuntimeUrlResolver, type RuntimeUrlResolver } from '@/lib/runtime-url';
|
||||
|
||||
const MAX_MARKDOWN_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_PREPARE_CACHE_ENTRIES = 1024;
|
||||
const NON_READY_CACHE_MS = 30_000;
|
||||
const SUPPORTED_IMAGE_MIME_TYPES = new Set([
|
||||
'image/png',
|
||||
'image/jpeg',
|
||||
@@ -9,37 +11,20 @@ const SUPPORTED_IMAGE_MIME_TYPES = new Set([
|
||||
'image/webp',
|
||||
]);
|
||||
|
||||
const parseLocalImagePath = (source: string): string => {
|
||||
let value = source;
|
||||
if (/^file:\/\//i.test(value)) {
|
||||
try {
|
||||
const fileUrl = new URL(value);
|
||||
if (fileUrl.protocol !== 'file:') return '';
|
||||
value = fileUrl.host && fileUrl.host !== 'localhost'
|
||||
? `//${fileUrl.host}${fileUrl.pathname}`
|
||||
: fileUrl.pathname;
|
||||
if (/^\/[A-Za-z]:\//.test(value)) value = value.slice(1);
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
export type PreparedMarkdownImage =
|
||||
| { status: 'ready'; path: string; outsideFileGrant?: string; expiresAt?: number }
|
||||
| { status: 'missing' | 'error' };
|
||||
|
||||
const path = value.split(/[?#]/, 1)[0] ?? '';
|
||||
try {
|
||||
return decodeURIComponent(path);
|
||||
} catch {
|
||||
return path;
|
||||
}
|
||||
type PrepareCacheEntry = {
|
||||
result: Map<string, PreparedMarkdownImage>;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
const blobToDataUrl = (blob: Blob): Promise<string> => new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => typeof reader.result === 'string'
|
||||
? resolve(reader.result)
|
||||
: reject(new Error('Unable to encode image'));
|
||||
reader.onerror = () => reject(reader.error ?? new Error('Unable to encode image'));
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
const prepareCaches = new WeakMap<RuntimeUrlResolver, Map<string, PrepareCacheEntry>>();
|
||||
|
||||
const throwIfAborted = (signal: AbortSignal): void => {
|
||||
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
|
||||
};
|
||||
|
||||
const hasImageSignature = async (blob: Blob, mimeType: string): Promise<boolean> => {
|
||||
const bytes = new Uint8Array(await blob.slice(0, 12).arrayBuffer());
|
||||
@@ -48,9 +33,7 @@ const hasImageSignature = async (blob: Blob, mimeType: string): Promise<boolean>
|
||||
return bytes[0] === 0x89 && ascii(1, 4) === 'PNG'
|
||||
&& bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a;
|
||||
}
|
||||
if (mimeType === 'image/jpeg') {
|
||||
return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
|
||||
}
|
||||
if (mimeType === 'image/jpeg') return bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff;
|
||||
if (mimeType === 'image/gif') {
|
||||
const gif = ascii(0, 6);
|
||||
return gif === 'GIF87a' || gif === 'GIF89a';
|
||||
@@ -67,66 +50,123 @@ const validateImageBlob = async (blob: Blob, mimeType: string): Promise<void> =>
|
||||
const validateDataImage = async (source: string): Promise<void> => {
|
||||
const match = /^data:(image\/(?:png|jpeg|gif|webp));base64,([\s\S]*)$/i.exec(source);
|
||||
if (!match?.[1] || match[2] === undefined) throw new Error('Invalid image data URL');
|
||||
const encoded = match[2];
|
||||
if (encoded.length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) {
|
||||
throw new Error('Image is too large');
|
||||
}
|
||||
|
||||
if (match[2].length > Math.ceil(MAX_MARKDOWN_IMAGE_BYTES * 4 / 3) + 4) throw new Error('Image is too large');
|
||||
let binary: string;
|
||||
try {
|
||||
binary = atob(encoded);
|
||||
binary = atob(match[2]);
|
||||
} catch {
|
||||
throw new Error('Invalid image data URL');
|
||||
}
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
|
||||
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
||||
await validateImageBlob(new Blob([bytes]), match[1].toLowerCase());
|
||||
};
|
||||
|
||||
export const isLocalMarkdownImageSource = (source: string): boolean => (
|
||||
!/^(?:https?:)?\/\//i.test(source)
|
||||
&& !/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)
|
||||
);
|
||||
|
||||
export const prepareLocalMarkdownImages = async ({
|
||||
sources,
|
||||
directory,
|
||||
sessionId,
|
||||
messageId,
|
||||
signal,
|
||||
}: {
|
||||
sources: readonly string[];
|
||||
directory: string;
|
||||
sessionId: string;
|
||||
messageId: string;
|
||||
signal: AbortSignal;
|
||||
}): Promise<Map<string, PreparedMarkdownImage>> => {
|
||||
const resolver = getRuntimeUrlResolver();
|
||||
let cache = prepareCaches.get(resolver);
|
||||
if (!cache) {
|
||||
cache = new Map();
|
||||
prepareCaches.set(resolver, cache);
|
||||
}
|
||||
const key = `${sessionId}\0${messageId}\0${directory}\0${sources.join('\0')}`;
|
||||
const cached = cache.get(key);
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
cache.delete(key);
|
||||
cache.set(key, cached);
|
||||
return cached.result;
|
||||
}
|
||||
if (cached) cache.delete(key);
|
||||
|
||||
const response = await runtimeFetch(
|
||||
`/api/openchamber/sessions/${encodeURIComponent(sessionId)}/markdown-image-grants`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ directory, messageId, sources }),
|
||||
signal,
|
||||
},
|
||||
);
|
||||
if (!response.ok) throw new Error(`Unable to prepare images (${response.status})`);
|
||||
const payload = await response.json() as {
|
||||
results?: Array<{
|
||||
source?: string;
|
||||
status?: string;
|
||||
path?: string;
|
||||
outsideFileGrant?: string;
|
||||
expiresAt?: number;
|
||||
}>;
|
||||
};
|
||||
const prepared = new Map<string, PreparedMarkdownImage>();
|
||||
for (const result of payload.results ?? []) {
|
||||
if (!result.source) continue;
|
||||
if (result.status === 'ready' && result.path) {
|
||||
prepared.set(result.source, {
|
||||
status: 'ready',
|
||||
path: result.path,
|
||||
outsideFileGrant: result.outsideFileGrant,
|
||||
expiresAt: result.expiresAt,
|
||||
});
|
||||
} else if (result.status === 'missing') {
|
||||
prepared.set(result.source, { status: 'missing' });
|
||||
} else {
|
||||
prepared.set(result.source, { status: 'error' });
|
||||
}
|
||||
}
|
||||
for (const source of sources) {
|
||||
if (!prepared.has(source)) prepared.set(source, { status: 'error' });
|
||||
}
|
||||
while (cache.size >= MAX_PREPARE_CACHE_ENTRIES) cache.delete(cache.keys().next().value!);
|
||||
const allReady = [...prepared.values()].every((value) => value.status === 'ready');
|
||||
const grantExpiry = Math.min(...[...prepared.values()]
|
||||
.filter((value): value is Extract<PreparedMarkdownImage, { status: 'ready' }> => value.status === 'ready')
|
||||
.map((value) => value.expiresAt ?? Number.POSITIVE_INFINITY));
|
||||
cache.set(key, {
|
||||
result: prepared,
|
||||
expiresAt: allReady ? grantExpiry : Date.now() + NON_READY_CACHE_MS,
|
||||
});
|
||||
return prepared;
|
||||
};
|
||||
|
||||
export const resolveMarkdownImageSource = async (
|
||||
source: string,
|
||||
directory: string,
|
||||
signal: AbortSignal,
|
||||
): Promise<string> => {
|
||||
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
|
||||
throwIfAborted(signal);
|
||||
if (/^(?:https?:)?\/\//i.test(source)) return source;
|
||||
|
||||
if (/^data:image\/(?:png|jpeg|gif|webp);base64,/i.test(source)) {
|
||||
await validateDataImage(source);
|
||||
if (signal.aborted) throw new DOMException('Image load aborted', 'AbortError');
|
||||
throwIfAborted(signal);
|
||||
return source;
|
||||
}
|
||||
|
||||
const localPath = parseLocalImagePath(source);
|
||||
const absolutePath = toAbsoluteFilePath(directory, localPath);
|
||||
if (!directory || !localPath || !isFilePathWithinDirectory(absolutePath, directory)) {
|
||||
throw new Error('Image path is outside the active workspace');
|
||||
}
|
||||
|
||||
const statResponse = await runtimeFetch('/api/fs/stat', {
|
||||
query: { path: absolutePath, directory, optional: 'true' },
|
||||
signal,
|
||||
});
|
||||
if (!statResponse.ok) throw new Error(`Unable to inspect image (${statResponse.status})`);
|
||||
const stat = await statResponse.json() as { isFile?: boolean; size?: number };
|
||||
if (!stat.isFile) throw new Error('Image path is not a file');
|
||||
if (typeof stat.size === 'number' && stat.size > MAX_MARKDOWN_IMAGE_BYTES) {
|
||||
throw new Error('Image is too large');
|
||||
}
|
||||
|
||||
const response = await runtimeFetch('/api/fs/raw', {
|
||||
query: { path: absolutePath, directory },
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) throw new Error(`Unable to load image (${response.status})`);
|
||||
|
||||
const mimeType = (response.headers.get('content-type') ?? '').split(';', 1)[0]?.toLowerCase() ?? '';
|
||||
const contentLength = Number(response.headers.get('content-length'));
|
||||
if (Number.isFinite(contentLength) && contentLength > MAX_MARKDOWN_IMAGE_BYTES) {
|
||||
throw new Error('Image is too large');
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
await validateImageBlob(blob, mimeType);
|
||||
return blobToDataUrl(blob);
|
||||
throw new Error('Local image has not been prepared');
|
||||
};
|
||||
|
||||
export const getPreparedMarkdownImageUrl = (
|
||||
image: Extract<PreparedMarkdownImage, { status: 'ready' }>,
|
||||
directory: string,
|
||||
): string => getRuntimeUrlResolver().authenticatedAsset(
|
||||
'/api/fs/raw',
|
||||
{
|
||||
path: image.path,
|
||||
directory,
|
||||
allowOutsideWorkspace: image.outsideFileGrant ? 'true' : undefined,
|
||||
outsideFileGrant: image.outsideFileGrant,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1876,7 +1876,6 @@ const AssistantMessageBody = React.memo(({
|
||||
chatRenderMode={chatRenderMode}
|
||||
onContentChange={onContentChange}
|
||||
onShowPopup={onShowPopup}
|
||||
enableMarkdownImages={isMessageCompleted}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
@@ -2031,7 +2030,6 @@ const AssistantMessageBody = React.memo(({
|
||||
collapsedPreviewCount,
|
||||
expandedTools,
|
||||
isMobile,
|
||||
isMessageCompleted,
|
||||
isActivityOwnerMessage,
|
||||
isSortedRenderMode,
|
||||
lastRenderableTextPartIndex,
|
||||
@@ -2244,6 +2242,8 @@ const AssistantMessageBody = React.memo(({
|
||||
</div>
|
||||
<MessageFilesDisplay files={parts} onShowPopup={onShowPopup} />
|
||||
<MarkdownImageGallery
|
||||
sessionId={sessionId}
|
||||
messageId={messageId}
|
||||
contents={finalizedAssistantMarkdownContents}
|
||||
onShowPopup={onShowPopup}
|
||||
/>
|
||||
|
||||
@@ -19,7 +19,6 @@ interface AssistantTextPartProps {
|
||||
chatRenderMode?: 'sorted' | 'live';
|
||||
onContentChange?: (reason?: ContentChangeReason, messageId?: string) => void;
|
||||
onShowPopup?: (content: ToolPopupContent) => void;
|
||||
enableMarkdownImages?: boolean;
|
||||
}
|
||||
|
||||
const AssistantTextPart: React.FC<AssistantTextPartProps> = ({
|
||||
@@ -28,7 +27,6 @@ const AssistantTextPart: React.FC<AssistantTextPartProps> = ({
|
||||
streamPhase,
|
||||
chatRenderMode = 'live',
|
||||
onShowPopup,
|
||||
enableMarkdownImages = false,
|
||||
}) => {
|
||||
// Use part directly from props — parent provides the latest version from the store.
|
||||
// No store subscription here to avoid re-render cascade from unrelated delta events.
|
||||
@@ -103,7 +101,6 @@ const AssistantTextPart: React.FC<AssistantTextPartProps> = ({
|
||||
disableStreamAnimation={chatRenderMode === 'sorted'}
|
||||
variant={part.type === 'reasoning' ? 'reasoning' : 'assistant'}
|
||||
enableFileReferences={isFinalized}
|
||||
enableLocalImages={enableMarkdownImages && !isStreaming && part.type === 'text'}
|
||||
onShowPopup={onShowPopup}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -55,23 +55,30 @@ Use this doc when you ask an agent to change tool/header/description behavior.
|
||||
HTML is sanitized as defense in depth, with script and style elements
|
||||
forbidden, so message content cannot inject active DOM or application-wide
|
||||
CSS into any runtime surface.
|
||||
- Final assistant Markdown collects HTTP(S), embedded, and workspace-local
|
||||
- Final assistant Markdown rendering is independent from image gallery
|
||||
extraction: gallery presence never changes the chat body. Assistant image
|
||||
syntax consistently renders as a shared image icon followed by its filename,
|
||||
without loading the image in the body; tool and simple Markdown retain normal
|
||||
inline image rendering. The gallery separately collects HTTP(S), embedded, and workspace-local
|
||||
PNG/JPEG/GIF/WebP image candidates into one 100px thumbnail gallery in the
|
||||
message-completion area after all message text and above the turn's changed
|
||||
files. Each muted filename caption includes the shared image-file icon.
|
||||
HTTP(S) images keep their browser URL. Embedded and workspace-local images
|
||||
are limited to 10 MiB, validated as PNG/JPEG/GIF/WebP, and local paths are
|
||||
fetched through the active runtime before conversion to data URLs. Local
|
||||
Markdown links whose target has one of
|
||||
those image suffixes stay links in the text and open the same existing
|
||||
full-screen image preview as the gallery; image syntax does not insert a
|
||||
large inline image. A
|
||||
HTTP(S) images keep their browser URL. Embedded and workspace-local gallery
|
||||
images are limited to 10 MiB and validated as PNG/JPEG/GIF/WebP. Local paths
|
||||
reuse the existing authenticated `/api/fs/raw` asset URL. Chat
|
||||
Markdown uses the assistant image-label policy without gallery-specific
|
||||
link rewriting, completion-state switching, or hidden placeholders. A
|
||||
completed assistant message hydrates at most 12 unique image candidates,
|
||||
including persisted text parts that omit their optional part-level end time.
|
||||
Thumbnail assets begin loading only when their gallery items approach the
|
||||
viewport, so mounted historical messages do not eagerly read every image.
|
||||
A gallery approaching the viewport prepares all local candidates in one
|
||||
message-level request, while each asset URL loads only when its own thumbnail
|
||||
approaches the viewport. Mounted historical messages therefore do not
|
||||
eagerly read every image.
|
||||
Gallery clicks do not introduce or alter preview chrome: desktop and mobile
|
||||
both reuse the pre-existing attachment image preview overlay.
|
||||
Workspace-external images receive the existing path-bound `outsideFileGrant`
|
||||
only when the server verifies the exact source in the owning assistant
|
||||
message and the real file is inside OpenCode's dedicated temporary directory.
|
||||
- `read` and `skill` are **static navigation tools** and render via `StaticToolRow`.
|
||||
- Every other tool, including search/fetch, OpenCode built-ins, custom tools, plugins, and MCP tools, is **expandable** and renders through `ToolPart`.
|
||||
- The managed `openchamber` plugin tool uses the expandable path and hides its broad protocol input. The plugin supplies the selected action's human description as the native tool title; the UI renders that metadata without owning an action map. The full versioned result envelope renders through the same neutral JSON summary/tree/raw views as other tools, without a tool-specific output card.
|
||||
|
||||
@@ -1151,12 +1151,6 @@ html:not(.dark) .chat-scroll {
|
||||
color: var(--markdown-link-hover, var(--primary));
|
||||
}
|
||||
|
||||
[data-openchamber-finalized-assistant-images="true"] [data-openchamber-markdown-image-placeholder="true"],
|
||||
[data-openchamber-finalized-assistant-images="true"] p:has(> [data-openchamber-markdown-image-placeholder="true"]:only-child),
|
||||
[data-openchamber-finalized-assistant-images="true"] li:has(> [data-openchamber-markdown-image-placeholder="true"]:only-child) {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.markdown-content [data-openchamber-file-link="true"] {
|
||||
color: var(--markdown-link, var(--primary));
|
||||
cursor: pointer;
|
||||
|
||||
@@ -45,6 +45,7 @@ The webview CSP permits `blob:` only for `worker-src` so shared UI parsers can r
|
||||
|
||||
- `bridge-localfs-proxy-runtime.ts`
|
||||
- Local `/api/fs/read` and `/api/fs/raw` proxy helpers and shared proxy utility helpers.
|
||||
- Returns an explicit unsupported response for server-owned Markdown image grants instead of forwarding them to OpenCode.
|
||||
|
||||
- `bridge-proxy-runtime.ts`
|
||||
- Proxy route handlers (`api:proxy`, `api:session:message`) with injected helper dependencies.
|
||||
|
||||
@@ -37,6 +37,14 @@ mock.module('vscode', () => ({
|
||||
const { tryHandleLocalFsProxy } = await import('./bridge-localfs-proxy-runtime');
|
||||
|
||||
describe('bridge local fs proxy', () => {
|
||||
it('does not forward server-owned Markdown image grant routes to OpenCode', async () => {
|
||||
const response = await tryHandleLocalFsProxy('POST', '/api/openchamber/sessions/ses_1/markdown-image-grants');
|
||||
|
||||
expect(response?.status).toBe(501);
|
||||
expect(Buffer.from(response?.bodyBase64 ?? '', 'base64').toString('utf8'))
|
||||
.toContain('not supported in the VS Code runtime');
|
||||
});
|
||||
|
||||
it('returns a quiet optional stat miss for missing files', async () => {
|
||||
const response = await tryHandleLocalFsProxy('GET', '/api/fs/stat?path=%2Fmissing.ts&optional=true');
|
||||
|
||||
|
||||
@@ -56,6 +56,9 @@ export const tryHandleLocalFsProxy = async (method: string, requestPath: string)
|
||||
}
|
||||
|
||||
const fsProxyPath = normalizeFsProxyPath(parsed.pathname);
|
||||
if (/^\/api\/openchamber\/sessions\/[^/]+\/markdown-image-grants$/.test(parsed.pathname)) {
|
||||
return buildProxyJsonError(501, 'Markdown image grants are not supported in the VS Code runtime');
|
||||
}
|
||||
if (!fsProxyPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# Markdown Image Grants
|
||||
|
||||
## Purpose
|
||||
|
||||
This module lets the Markdown image gallery display images that an assistant
|
||||
explicitly referenced from OpenCode's temporary directory when the UI is on a
|
||||
different machine.
|
||||
|
||||
## Contract
|
||||
|
||||
- Chat Markdown rendering is independent: assistant image syntax renders as an
|
||||
icon and filename, while the gallery only reads finalized Markdown to collect
|
||||
image candidates.
|
||||
- `POST /api/openchamber/sessions/:sessionId/markdown-image-grants` prepares up to 12
|
||||
local images in one message-level request. The server fetches the assistant
|
||||
message once and verifies every exact image source before reading files.
|
||||
- Relative and workspace-contained absolute paths resolve against the active
|
||||
directory. Other absolute paths are accepted only inside
|
||||
`os.tmpdir()/opencode` after `realpath` resolution.
|
||||
- PNG, JPEG, GIF, and WebP files are signature-checked and limited to 10 MiB.
|
||||
- Prepare requests inspect only file metadata and signatures. Workspace images
|
||||
reuse the existing authenticated `/api/fs/raw` asset route directly. Images
|
||||
under `os.tmpdir()/opencode` receive the existing path-bound `raw`
|
||||
`outsideFileGrant`; this module does not add another asset lifetime, copy, or
|
||||
storage layer. Missing files return per-source results so the gallery can
|
||||
remove only those items.
|
||||
|
||||
The routes are OpenChamber-owned and must be registered before the generic
|
||||
OpenCode proxy. Web, Electron, hosted mobile, and Capacitor use the shared
|
||||
server implementation. VS Code returns an explicit unsupported response.
|
||||
@@ -0,0 +1,215 @@
|
||||
import express from 'express';
|
||||
import { constants as fsConstants } from 'node:fs';
|
||||
import { mintOutsideFileGrant } from '../fs/routes.js';
|
||||
|
||||
const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_IMAGE_SOURCES = 12;
|
||||
|
||||
const asString = (value) => typeof value === 'string' ? value.trim() : '';
|
||||
|
||||
const isWithin = (target, root, path) => {
|
||||
const relative = path.relative(root, target);
|
||||
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
|
||||
};
|
||||
|
||||
const parseFileSource = (source) => {
|
||||
if (/^file:\/\//i.test(source)) {
|
||||
try {
|
||||
const url = new URL(source);
|
||||
if (url.protocol !== 'file:' || (url.host && url.host !== 'localhost')) return '';
|
||||
const pathname = decodeURIComponent(url.pathname);
|
||||
return /^\/[A-Za-z]:\//.test(pathname) ? pathname.slice(1) : pathname;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
const pathname = source.split(/[?#]/, 1)[0] || '';
|
||||
try {
|
||||
return decodeURIComponent(pathname);
|
||||
} catch {
|
||||
return pathname;
|
||||
}
|
||||
};
|
||||
|
||||
const hasImageSignature = (bytes) => {
|
||||
if (bytes.length >= 8
|
||||
&& bytes[0] === 0x89 && bytes.subarray(1, 4).toString('ascii') === 'PNG'
|
||||
&& bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) return true;
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return true;
|
||||
const header = bytes.subarray(0, 12).toString('ascii');
|
||||
return header.startsWith('GIF87a')
|
||||
|| header.startsWith('GIF89a')
|
||||
|| (header.startsWith('RIFF') && header.slice(8, 12) === 'WEBP');
|
||||
};
|
||||
|
||||
const markdownImageSources = (message) => {
|
||||
const sources = new Set();
|
||||
for (const part of Array.isArray(message?.parts) ? message.parts : []) {
|
||||
if (part?.type !== 'text' || typeof part.text !== 'string') continue;
|
||||
// Code examples must never authorize file access, even when they contain image syntax.
|
||||
let fenced = false;
|
||||
for (const line of part.text.split('\n')) {
|
||||
if (/^\s{0,3}(?:```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
continue;
|
||||
}
|
||||
if (fenced) continue;
|
||||
const visible = line.replace(/`+[^`]*`+/g, '');
|
||||
const pattern = /(?<!\\)!\[[^\]]*]\(\s*(?:<([^>\n]+)>|([^\s)\n]+))/g;
|
||||
let match = pattern.exec(visible);
|
||||
while (match) {
|
||||
sources.add(match[1] || match[2]);
|
||||
match = pattern.exec(visible);
|
||||
}
|
||||
}
|
||||
}
|
||||
return sources;
|
||||
};
|
||||
|
||||
const fetchMessage = async ({ sessionId, messageId, directory, buildOpenCodeUrl, getOpenCodeAuthHeaders }) => {
|
||||
const url = new URL(buildOpenCodeUrl(
|
||||
`/session/${encodeURIComponent(sessionId)}/message/${encodeURIComponent(messageId)}`,
|
||||
'',
|
||||
));
|
||||
url.searchParams.set('directory', directory);
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
'x-opencode-directory': directory,
|
||||
...getOpenCodeAuthHeaders(),
|
||||
},
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) throw new Error(`OpenCode returned ${response.status}`);
|
||||
const message = await response.json().catch(() => null);
|
||||
return message?.info && Array.isArray(message.parts) ? message : null;
|
||||
};
|
||||
|
||||
const inspectImage = async ({ source, directory, approvedTempRoot, fsPromises, path }) => {
|
||||
const parsed = parseFileSource(source);
|
||||
if (!parsed) return { status: 'error' };
|
||||
const sourcePath = path.isAbsolute(parsed) ? parsed : path.resolve(directory, parsed);
|
||||
const workspaceRoot = path.resolve(directory);
|
||||
const outsideWorkspace = !isWithin(path.resolve(sourcePath), workspaceRoot, path);
|
||||
const root = outsideWorkspace ? approvedTempRoot : workspaceRoot;
|
||||
|
||||
try {
|
||||
// Resolve symlinks before comparing roots; lexical prefixes are not an authorization boundary.
|
||||
const [canonicalRoot, canonicalPath] = await Promise.all([
|
||||
fsPromises.realpath(root),
|
||||
fsPromises.realpath(sourcePath),
|
||||
]);
|
||||
if (!isWithin(canonicalPath, canonicalRoot, path)) return { status: 'error' };
|
||||
const handle = await fsPromises.open(canonicalPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
||||
try {
|
||||
const stats = await handle.stat();
|
||||
if (!stats.isFile() || stats.size > MAX_IMAGE_BYTES) return { status: 'error' };
|
||||
const header = Buffer.alloc(12);
|
||||
const { bytesRead } = await handle.read(header, 0, header.length, 0);
|
||||
if (!hasImageSignature(header.subarray(0, bytesRead))) return { status: 'error' };
|
||||
return {
|
||||
status: 'ready',
|
||||
path: outsideWorkspace ? canonicalPath : path.resolve(sourcePath),
|
||||
outsideWorkspace,
|
||||
};
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return { status: 'missing' };
|
||||
if (error?.code === 'EACCES' || error?.code === 'EPERM' || error?.code === 'ELOOP') {
|
||||
return { status: 'error' };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const registerMarkdownImageGrantRoutes = (app, dependencies) => {
|
||||
const {
|
||||
fsPromises,
|
||||
path,
|
||||
os,
|
||||
crypto,
|
||||
validateDirectoryPath,
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders,
|
||||
approvedTempRoot = path.join(os.tmpdir(), 'opencode'),
|
||||
} = dependencies;
|
||||
|
||||
app.post(
|
||||
'/api/openchamber/sessions/:sessionId/markdown-image-grants',
|
||||
express.json({ limit: '32kb' }),
|
||||
async (req, res) => {
|
||||
const sessionId = asString(req.params.sessionId);
|
||||
const messageId = asString(req.body?.messageId);
|
||||
const sources = Array.isArray(req.body?.sources)
|
||||
? [...new Set(req.body.sources.map(asString).filter(Boolean))]
|
||||
: [];
|
||||
if (!sessionId || !messageId || sources.length === 0 || sources.length > MAX_IMAGE_SOURCES) {
|
||||
return res.status(400).json({ error: 'sessionId, messageId, and 1-12 sources are required' });
|
||||
}
|
||||
const validatedDirectory = await validateDirectoryPath(asString(req.body?.directory));
|
||||
if (!validatedDirectory.ok) {
|
||||
return res.status(400).json({ error: validatedDirectory.error || 'Invalid directory' });
|
||||
}
|
||||
|
||||
try {
|
||||
const message = await fetchMessage({
|
||||
sessionId,
|
||||
messageId,
|
||||
directory: validatedDirectory.directory,
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders,
|
||||
});
|
||||
if (!message || message.info?.id !== messageId || message.info?.role !== 'assistant') {
|
||||
return res.status(404).json({ error: 'Assistant message not found' });
|
||||
}
|
||||
// Assistant text is authoritative: a remote client cannot mint grants for unreferenced paths.
|
||||
const referenced = markdownImageSources(message);
|
||||
const results = [];
|
||||
for (const source of sources) {
|
||||
if (!referenced.has(source)) {
|
||||
results.push({ source, status: 'error' });
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const inspected = await inspectImage({
|
||||
source,
|
||||
directory: validatedDirectory.directory,
|
||||
approvedTempRoot,
|
||||
fsPromises,
|
||||
path,
|
||||
});
|
||||
if (inspected.status !== 'ready') {
|
||||
results.push({ source, status: inspected.status });
|
||||
continue;
|
||||
}
|
||||
// Reuse the existing path-bound raw-file grant instead of creating another asset lifecycle.
|
||||
const grant = inspected.outsideWorkspace
|
||||
? await mintOutsideFileGrant(inspected.path, {
|
||||
scopes: ['raw'],
|
||||
fsPromises,
|
||||
path,
|
||||
crypto,
|
||||
})
|
||||
: null;
|
||||
results.push({
|
||||
source,
|
||||
status: 'ready',
|
||||
path: inspected.path,
|
||||
outsideFileGrant: grant?.outsideFileGrant,
|
||||
expiresAt: grant?.expiresAt,
|
||||
});
|
||||
} catch {
|
||||
results.push({ source, status: 'error' });
|
||||
}
|
||||
}
|
||||
return res.json({ results });
|
||||
} catch (error) {
|
||||
console.warn('[MarkdownImageGrants] failed to prepare images:', error?.message || error);
|
||||
return res.status(503).json({ error: 'Failed to prepare session images' });
|
||||
}
|
||||
},
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,175 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import express from 'express';
|
||||
import request from 'supertest';
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { registerMarkdownImageGrantRoutes } from './routes.js';
|
||||
|
||||
const PNG = Buffer.from(
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==',
|
||||
'base64',
|
||||
);
|
||||
const roots = [];
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllGlobals();
|
||||
await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
const createFixture = async ({ sources, markdown } = {}) => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'openchamber-session-assets-'));
|
||||
roots.push(root);
|
||||
const approvedTempRoot = path.join(root, 'opencode');
|
||||
const directory = path.join(root, 'workspace');
|
||||
await Promise.all([
|
||||
fs.mkdir(approvedTempRoot, { recursive: true }),
|
||||
fs.mkdir(directory, { recursive: true }),
|
||||
]);
|
||||
const defaultPath = path.join(approvedTempRoot, 'image.png');
|
||||
await fs.writeFile(defaultPath, PNG);
|
||||
const requestedSources = sources ?? [new URL(`file://${defaultPath}`).toString()];
|
||||
const text = markdown ?? requestedSources.map((source) => ``).join('\n');
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify({
|
||||
info: { id: 'msg_1', role: 'assistant' },
|
||||
parts: [{ type: 'text', text }],
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
let fullReadCount = 0;
|
||||
const app = express();
|
||||
registerMarkdownImageGrantRoutes(app, {
|
||||
fsPromises: {
|
||||
...fs,
|
||||
readFile: async (...args) => {
|
||||
fullReadCount += 1;
|
||||
return fs.readFile(...args);
|
||||
},
|
||||
},
|
||||
path,
|
||||
os,
|
||||
crypto,
|
||||
approvedTempRoot,
|
||||
validateDirectoryPath: async (candidate) => candidate === directory
|
||||
? { ok: true, directory }
|
||||
: { ok: false, error: 'Invalid directory' },
|
||||
buildOpenCodeUrl: (route) => `http://opencode.test${route}`,
|
||||
getOpenCodeAuthHeaders: () => ({ authorization: 'Basic test' }),
|
||||
});
|
||||
return {
|
||||
app,
|
||||
approvedTempRoot,
|
||||
directory,
|
||||
fetchMock,
|
||||
fullReadCount: () => fullReadCount,
|
||||
root,
|
||||
sources: requestedSources,
|
||||
};
|
||||
};
|
||||
|
||||
const prepare = (app, directory, sources) => request(app)
|
||||
.post('/api/openchamber/sessions/ses_1/markdown-image-grants')
|
||||
.send({ directory, messageId: 'msg_1', sources })
|
||||
.expect(200);
|
||||
|
||||
describe('session image assets', () => {
|
||||
it('prepares workspace and OpenCode temporary images with one message fetch', async () => {
|
||||
const fixture = await createFixture({ sources: ['workspace.png'] });
|
||||
await fs.writeFile(path.join(fixture.directory, 'workspace.png'), PNG);
|
||||
const temporaryPath = path.join(fixture.approvedTempRoot, 'temporary.png');
|
||||
await fs.writeFile(temporaryPath, PNG);
|
||||
const temporarySource = new URL(`file://${temporaryPath}`).toString();
|
||||
fixture.fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
info: { id: 'msg_1', role: 'assistant' },
|
||||
parts: [{ type: 'text', text: `\n` }],
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } }));
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, ['workspace.png', temporarySource]);
|
||||
|
||||
expect(fixture.fetchMock).toHaveBeenCalledTimes(1);
|
||||
expect(fixture.fullReadCount()).toBe(0);
|
||||
expect(response.body.results).toHaveLength(2);
|
||||
const canonicalTemporaryPath = await fs.realpath(temporaryPath);
|
||||
expect(response.body.results[0]).toEqual({
|
||||
source: 'workspace.png',
|
||||
status: 'ready',
|
||||
path: path.join(fixture.directory, 'workspace.png'),
|
||||
});
|
||||
expect(response.body.results[1]).toEqual(expect.objectContaining({
|
||||
source: temporarySource,
|
||||
status: 'ready',
|
||||
path: canonicalTemporaryPath,
|
||||
outsideFileGrant: expect.any(String),
|
||||
expiresAt: expect.any(Number),
|
||||
}));
|
||||
});
|
||||
|
||||
it('returns partial results without letting one missing image block valid images', async () => {
|
||||
const fixture = await createFixture({ sources: ['present.png', 'deleted.png'] });
|
||||
await fs.writeFile(path.join(fixture.directory, 'present.png'), PNG);
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, fixture.sources);
|
||||
|
||||
expect(response.body.results).toEqual([
|
||||
expect.objectContaining({ source: 'present.png', status: 'ready' }),
|
||||
{ source: 'deleted.png', status: 'missing' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('resolves encoded workspace paths without treating query or fragment text as a filename', async () => {
|
||||
const source = 'screen%20shot.png?version=1#preview';
|
||||
const fixture = await createFixture({ sources: [source] });
|
||||
await fs.writeFile(path.join(fixture.directory, 'screen shot.png'), PNG);
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, fixture.sources);
|
||||
|
||||
expect(response.body.results).toEqual([
|
||||
expect.objectContaining({ source, status: 'ready' }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects a source that the message does not reference', async () => {
|
||||
const fixture = await createFixture({ markdown: 'No image here.' });
|
||||
const response = await prepare(fixture.app, fixture.directory, fixture.sources);
|
||||
expect(response.body.results).toEqual([{ source: fixture.sources[0], status: 'error' }]);
|
||||
});
|
||||
|
||||
it('does not authorize image syntax inside fenced or inline code', async () => {
|
||||
const fixture = await createFixture({
|
||||
markdown: '```md\n\n```\n``',
|
||||
});
|
||||
const sources = ['FENCED', 'INLINE'];
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, sources);
|
||||
|
||||
expect(response.body.results).toEqual(sources.map((source) => ({ source, status: 'error' })));
|
||||
});
|
||||
|
||||
it('rejects paths outside the workspace and approved temporary root', async () => {
|
||||
const fixture = await createFixture();
|
||||
const outsidePath = path.join(fixture.root, 'outside.png');
|
||||
await fs.writeFile(outsidePath, PNG);
|
||||
const source = new URL(`file://${outsidePath}`).toString();
|
||||
fixture.fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({
|
||||
info: { id: 'msg_1', role: 'assistant' },
|
||||
parts: [{ type: 'text', text: `` }],
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } }));
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, [source]);
|
||||
expect(response.body.results).toEqual([{ source, status: 'error' }]);
|
||||
});
|
||||
|
||||
it('rejects non-image bytes and symlink escapes per source', async () => {
|
||||
const fixture = await createFixture({ sources: ['invalid.png', 'linked.png'] });
|
||||
await fs.writeFile(path.join(fixture.directory, 'invalid.png'), 'not an image');
|
||||
await fs.writeFile(path.join(fixture.root, 'outside.png'), PNG);
|
||||
await fs.symlink(path.join(fixture.root, 'outside.png'), path.join(fixture.directory, 'linked.png'));
|
||||
|
||||
const response = await prepare(fixture.app, fixture.directory, fixture.sources);
|
||||
expect(response.body.results).toEqual([
|
||||
{ source: 'invalid.png', status: 'error' },
|
||||
{ source: 'linked.png', status: 'error' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,7 @@ import { registerProjectIconRoutes } from './project-icon-routes.js';
|
||||
import { registerScheduledTaskRoutes } from '../scheduled-tasks/routes.js';
|
||||
import { registerOpenChamberSessionRoutes } from '../openchamber-sessions/routes.js';
|
||||
import { registerOpenChamberControlRoutes } from '../openchamber-control/routes.js';
|
||||
import { registerMarkdownImageGrantRoutes } from '../markdown-image-grants/routes.js';
|
||||
import { registerSkillRoutes } from './skill-routes.js';
|
||||
import { registerPluginRoutes } from './plugin-routes.js';
|
||||
import { getNpmInfo, clearCache as clearNpmCache } from './npm-registry.js';
|
||||
@@ -190,6 +191,16 @@ export const createFeatureRoutesRuntime = (dependencies) => {
|
||||
|
||||
registerOpenChamberControlRoutes(app, { controlService: openChamberControlService });
|
||||
|
||||
registerMarkdownImageGrantRoutes(app, {
|
||||
fsPromises,
|
||||
path,
|
||||
os,
|
||||
crypto,
|
||||
validateDirectoryPath,
|
||||
buildOpenCodeUrl,
|
||||
getOpenCodeAuthHeaders,
|
||||
});
|
||||
|
||||
registerConfigEntityRoutes(app, {
|
||||
resolveProjectDirectory,
|
||||
resolveOptionalProjectDirectory,
|
||||
|
||||
Reference in New Issue
Block a user