diff --git a/packages/ui/src/components/chat/ChatInput.tsx b/packages/ui/src/components/chat/ChatInput.tsx index 86f7e7e2..46f7e428 100644 --- a/packages/ui/src/components/chat/ChatInput.tsx +++ b/packages/ui/src/components/chat/ChatInput.tsx @@ -11,7 +11,12 @@ import { useAutoReviewStore } from '@/stores/useAutoReviewStore'; import { useSessionUIStore } from '@/sync/session-ui-store'; import { useSelectionStore } from '@/sync/selection-store'; import { useInputStore } from '@/sync/input-store'; -import { ACCEPTED_ATTACHMENT_EXTENSIONS, ATTACHMENT_ACCEPT } from '@/sync/attachment-files'; +import { + ACCEPTED_ATTACHMENT_EXTENSIONS, + ATTACHMENT_ACCEPT, + getUnsupportedAttachmentInputs, + type AttachmentInputModality, +} from '@/sync/attachment-files'; import type { AttachedFile } from '@/stores/types/sessionTypes'; import * as sessionActions from '@/sync/session-actions'; import { useDirectorySync, useUserMessageHistory } from '@/sync/sync-context'; @@ -1094,6 +1099,13 @@ const ChatInputComponent: React.FC = ({ onOpenSettings, scrollTo const currentProviderId = useConfigStore((state) => state.currentProviderId); const currentModelId = useConfigStore((state) => state.currentModelId); + const getModelMetadata = useConfigStore((state) => state.getModelMetadata); + // Subscribe to both sources read by getModelMetadata so async metadata and provider updates are observed. + useConfigStore((state) => state.modelsMetadata); + useConfigStore((state) => state.providers); + const currentModelMetadata = currentProviderId && currentModelId + ? getModelMetadata(currentProviderId, currentModelId) + : undefined; const currentVariant = useConfigStore((state) => state.currentVariant); const currentAgentName = useConfigStore((state) => state.currentAgentName); const setAgent = useConfigStore((state) => state.setAgent); @@ -1129,6 +1141,53 @@ const ChatInputComponent: React.FC = ({ onOpenSettings, scrollTo title: '', content: '', }); + const attachmentCompatibilityRef = React.useRef({ + modelKey: `${currentProviderId ?? ''}/${currentModelId ?? ''}`, + modalitySignature: currentModelMetadata?.modalities?.input?.slice().sort().join(',') ?? null, + attachmentIds: new Set(), + }); + + React.useEffect(() => { + const modelKey = `${currentProviderId ?? ''}/${currentModelId ?? ''}`; + const inputModalities = currentModelMetadata?.modalities?.input; + const modalitySignature = inputModalities?.slice().sort().join(',') ?? null; + const previous = attachmentCompatibilityRef.current; + const modelChanged = previous.modelKey !== modelKey; + const metadataBecameAvailable = previous.modalitySignature === null && modalitySignature !== null; + const filesToCheck = modelChanged || metadataBecameAvailable + ? attachedFiles + : attachedFiles.filter((file) => !previous.attachmentIds.has(file.id)); + + attachmentCompatibilityRef.current = { + modelKey, + modalitySignature, + attachmentIds: new Set(attachedFiles.map((file) => file.id)), + }; + + if (!inputModalities || filesToCheck.length === 0) return; + + const incompatibleFiles = getUnsupportedAttachmentInputs(filesToCheck, inputModalities); + if (incompatibleFiles.length === 0) return; + + const unsupportedModalities = Array.from(new Set(incompatibleFiles.map(({ modality }) => modality))); + const modalityLabels: Record = { + text: t('chat.modelControls.modality.text'), + image: t('chat.modelControls.modality.image'), + pdf: t('chat.modelControls.modality.pdf'), + audio: t('chat.modelControls.modality.audio'), + video: t('chat.modelControls.modality.video'), + }; + const filenames = incompatibleFiles.map(({ attachment }) => attachment.filename); + const fileSummary = filenames.length > 3 + ? `${filenames.slice(0, 3).join(', ')} (+${filenames.length - 3})` + : filenames.join(', '); + + toast.warning(t('chat.chatInput.toast.unsupportedAttachmentModalities', { + model: currentModelMetadata.name ?? currentModelId ?? '', + modalities: unsupportedModalities.map((modality) => modalityLabels[modality]).join(', '), + files: fileSummary, + }), { id: `attachment-modalities:${modelKey}` }); + }, [attachedFiles, currentModelId, currentModelMetadata, currentProviderId, t]); const handleShowAttachmentPreview = React.useCallback((content: ToolPopupContent) => { if (!content.image) return; diff --git a/packages/ui/src/lib/i18n/messages/en.ts b/packages/ui/src/lib/i18n/messages/en.ts index 83036ff8..f7116048 100644 --- a/packages/ui/src/lib/i18n/messages/en.ts +++ b/packages/ui/src/lib/i18n/messages/en.ts @@ -2002,6 +2002,7 @@ export const dict = { 'chat.chatInput.toast.addedFileMentions': 'Added {count} file mention(s)', 'chat.chatInput.toast.attachFileFailed': 'Failed to attach file', 'chat.chatInput.toast.attachNamedFailed': 'Failed to attach {name}', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model} does not support {modalities} input required by {files}. You can still send the message, but these attachments may be ignored.', 'chat.chatInput.toast.someFilesSkipped': 'Some files were skipped:\n{summary}', 'chat.chatInput.toast.vscodePickFailed': 'Failed to pick files in VS Code', 'chat.chatInput.toast.openSessionFirst': 'Open a session first', diff --git a/packages/ui/src/lib/i18n/messages/es.ts b/packages/ui/src/lib/i18n/messages/es.ts index a888528a..44692e8e 100644 --- a/packages/ui/src/lib/i18n/messages/es.ts +++ b/packages/ui/src/lib/i18n/messages/es.ts @@ -1968,6 +1968,7 @@ export const dict: Record = { "chat.chatInput.toast.addedFileMentions": "Se añadieron {count} mención(es) de archivo", "chat.chatInput.toast.attachFileFailed": "No se pudo adjuntar el archivo", "chat.chatInput.toast.attachNamedFailed": "No se pudo adjuntar {name}", + "chat.chatInput.toast.unsupportedAttachmentModalities": "{model} no admite la entrada de {modalities} necesaria para {files}. Aún puedes enviar el mensaje, pero estos adjuntos podrían ignorarse.", "chat.chatInput.toast.someFilesSkipped": "Algunos archivos se omitieron:\n{summary}", "chat.chatInput.toast.vscodePickFailed": "No se pudieron seleccionar archivos en VS Code", "chat.chatInput.toast.openSessionFirst": "Abre una sesión primero", diff --git a/packages/ui/src/lib/i18n/messages/fr.ts b/packages/ui/src/lib/i18n/messages/fr.ts index f8f6ccdb..8eae1a30 100644 --- a/packages/ui/src/lib/i18n/messages/fr.ts +++ b/packages/ui/src/lib/i18n/messages/fr.ts @@ -1780,6 +1780,7 @@ export const dict = { 'chat.chatInput.toast.addedFileMentions': 'Ajout des mentions du fichier {count}', 'chat.chatInput.toast.attachFileFailed': 'Impossible de joindre le fichier', 'chat.chatInput.toast.attachNamedFailed': 'Échec de la connexion du {name}', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model} ne prend pas en charge l’entrée {modalities} requise par {files}. Vous pouvez tout de même envoyer le message, mais ces pièces jointes risquent d’être ignorées.', 'chat.chatInput.toast.someFilesSkipped': 'Certains fichiers ont été ignorés :\n{summary}', 'chat.chatInput.toast.vscodePickFailed': 'Échec de la sélection des fichiers dans VS Code', 'chat.chatInput.toast.openSessionFirst': 'Ouvrir d\'abord une session', diff --git a/packages/ui/src/lib/i18n/messages/ja.ts b/packages/ui/src/lib/i18n/messages/ja.ts index 11e921e5..61094652 100644 --- a/packages/ui/src/lib/i18n/messages/ja.ts +++ b/packages/ui/src/lib/i18n/messages/ja.ts @@ -2001,6 +2001,7 @@ export const dict: Record = { 'gitView.commit.aiHighlights.insertTooltip': '挿入のツールチップ', 'gitView.commit.commitAria': 'コミットのariaラベル', 'chat.chatInput.toast.attachNamedFailed': '{name}の添付に失敗しました', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model} は {files} に必要な {modalities} 入力をサポートしていません。メッセージは送信できますが、これらの添付ファイルは無視される可能性があります。', 'chat.chatInput.toast.someFilesSkipped': '一部のファイルがスキップされました:\n{summary}', 'chat.chatInput.toast.vscodePickFailed': 'VS Codeでのファイル選択に失敗しました', 'chat.chatInput.toast.openSessionFirst': '先にセッションを開いてください', diff --git a/packages/ui/src/lib/i18n/messages/ko.ts b/packages/ui/src/lib/i18n/messages/ko.ts index e238234b..3362e16a 100644 --- a/packages/ui/src/lib/i18n/messages/ko.ts +++ b/packages/ui/src/lib/i18n/messages/ko.ts @@ -2002,6 +2002,7 @@ export const dict: Record = { 'chat.chatInput.toast.addedFileMentions': '파일 멘션 {count}개 추가됨', 'chat.chatInput.toast.attachFileFailed': '첨부 파일 실패', 'chat.chatInput.toast.attachNamedFailed': '첨부 {name} 실패', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model}은(는) {files}에 필요한 {modalities} 입력을 지원하지 않습니다. 메시지를 보낼 수는 있지만 해당 첨부 파일이 무시될 수 있습니다.', 'chat.chatInput.toast.someFilesSkipped': '일부 파일을 건너뛰었습니다:\n{summary}', 'chat.chatInput.toast.vscodePickFailed': 'VS Code에서 파일 선택에 실패했습니다', 'chat.chatInput.toast.openSessionFirst': '먼저 세션을 여세요', diff --git a/packages/ui/src/lib/i18n/messages/pl.ts b/packages/ui/src/lib/i18n/messages/pl.ts index 86bbbeff..bab81d84 100644 --- a/packages/ui/src/lib/i18n/messages/pl.ts +++ b/packages/ui/src/lib/i18n/messages/pl.ts @@ -1169,6 +1169,7 @@ export const dict: Record = { 'chat.chatInput.toast.addedFileMentions': 'Dodano {count} wzmianek o plikach', 'chat.chatInput.toast.attachFileFailed': 'Nie udało się dołączyć pliku', 'chat.chatInput.toast.attachNamedFailed': 'Nie udało się dołączyć {name}', + 'chat.chatInput.toast.unsupportedAttachmentModalities': 'Model {model} nie obsługuje danych wejściowych {modalities} wymaganych przez {files}. Nadal możesz wysłać wiadomość, ale te załączniki mogą zostać zignorowane.', 'chat.chatInput.toast.attachmentsTooLarge': 'Załączniki są zbyt duże, aby je wysłać. Spróbuj zmniejszyć liczbę lub rozmiar obrazów.', 'chat.chatInput.toast.clipboardAttachFailed': 'Nie udało się dołączyć obrazu ze schowka', 'chat.chatInput.toast.compactFailed': 'Nie udało się skompaktować sesji', diff --git a/packages/ui/src/lib/i18n/messages/pt-BR.ts b/packages/ui/src/lib/i18n/messages/pt-BR.ts index eedfb214..51e2ac27 100644 --- a/packages/ui/src/lib/i18n/messages/pt-BR.ts +++ b/packages/ui/src/lib/i18n/messages/pt-BR.ts @@ -1968,6 +1968,7 @@ export const dict: Record = { "chat.chatInput.toast.addedFileMentions": "Foram adicionadas {count} menção(es) de arquivo", "chat.chatInput.toast.attachFileFailed": "Não foi possível anexar o arquivo", "chat.chatInput.toast.attachNamedFailed": "Não foi possível anexar {name}", + "chat.chatInput.toast.unsupportedAttachmentModalities": "{model} não oferece suporte à entrada de {modalities} exigida por {files}. Você ainda pode enviar a mensagem, mas esses anexos podem ser ignorados.", "chat.chatInput.toast.someFilesSkipped": "Alguns arquivos foram omitidos:\n{summary}", "chat.chatInput.toast.vscodePickFailed": "Não foi possível selecionar arquivos em VS Code", "chat.chatInput.toast.openSessionFirst": "Abra uma sessão primeiro", diff --git a/packages/ui/src/lib/i18n/messages/uk.ts b/packages/ui/src/lib/i18n/messages/uk.ts index 74aaf9c5..d212748f 100644 --- a/packages/ui/src/lib/i18n/messages/uk.ts +++ b/packages/ui/src/lib/i18n/messages/uk.ts @@ -1968,6 +1968,7 @@ export const dict: Record = { "chat.chatInput.toast.addedFileMentions": "Додано згадки файлів {count}", "chat.chatInput.toast.attachFileFailed": "Не вдалося прикріпити файл", "chat.chatInput.toast.attachNamedFailed": "Не вдалося прикріпити {name}", + "chat.chatInput.toast.unsupportedAttachmentModalities": "{model} не підтримує вхідні дані {modalities}, потрібні для {files}. Повідомлення все одно можна надіслати, але ці вкладення можуть бути проігноровані.", "chat.chatInput.toast.someFilesSkipped": "Деякі файли були пропущені:\n{summary}", "chat.chatInput.toast.vscodePickFailed": "Не вдалося вибрати файли в VS Code", "chat.chatInput.toast.openSessionFirst": "Спочатку відкрийте сесію", diff --git a/packages/ui/src/lib/i18n/messages/zh-CN.ts b/packages/ui/src/lib/i18n/messages/zh-CN.ts index 0926edc8..ee3e0f90 100644 --- a/packages/ui/src/lib/i18n/messages/zh-CN.ts +++ b/packages/ui/src/lib/i18n/messages/zh-CN.ts @@ -1968,6 +1968,7 @@ export const dict: Record = { 'chat.chatInput.toast.addedFileMentions': '已添加 {count} 个文件提及', 'chat.chatInput.toast.attachFileFailed': '附加文件失败', 'chat.chatInput.toast.attachNamedFailed': '附加 {name} 失败', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model} 不支持 {files} 所需的 {modalities} 输入。你仍可发送消息，但这些附件可能会被忽略。', 'chat.chatInput.toast.someFilesSkipped': '部分文件被跳过：\n{summary}', 'chat.chatInput.toast.vscodePickFailed': '在 VS Code 中选择文件失败', 'chat.chatInput.toast.openSessionFirst': '请先打开一个会话', diff --git a/packages/ui/src/lib/i18n/messages/zh-TW.ts b/packages/ui/src/lib/i18n/messages/zh-TW.ts index 10d19522..c3096029 100644 --- a/packages/ui/src/lib/i18n/messages/zh-TW.ts +++ b/packages/ui/src/lib/i18n/messages/zh-TW.ts @@ -1972,6 +1972,7 @@ export const dict: Record = { 'chat.chatInput.toast.addedFileMentions': '已加入 {count} 個檔案提及', 'chat.chatInput.toast.attachFileFailed': '附加檔案失敗', 'chat.chatInput.toast.attachNamedFailed': '附加 {name} 失敗', + 'chat.chatInput.toast.unsupportedAttachmentModalities': '{model} 不支援 {files} 所需的 {modalities} 輸入。你仍可傳送訊息，但這些附件可能會被忽略。', 'chat.chatInput.toast.someFilesSkipped': '部分檔案被跳過：\n{summary}', 'chat.chatInput.toast.vscodePickFailed': '在 VS Code 中選擇檔案失敗', 'chat.chatInput.toast.openSessionFirst': '請先開啟一個會話', diff --git a/packages/ui/src/sync/DOCUMENTATION.md b/packages/ui/src/sync/DOCUMENTATION.md index 747bdc86..56bcfff8 100644 --- a/packages/ui/src/sync/DOCUMENTATION.md +++ b/packages/ui/src/sync/DOCUMENTATION.md @@ -57,6 +57,8 @@ Local chat attachments are normalized by `attachment-files.ts` before entering ` Office and OpenDocument packages are metadata-validated before asynchronous extraction, with limits of 20 MB compressed input, 5,000 archive entries, 25 MB per entry, 8 MB per XML part, and 100 MB total uncompressed content. Unsafe or non-canonical archive paths reject the whole attachment, and only XML, relationship, and supported image entries are decompressed and retained. Extracted text, including its explicit truncation notice, is bounded to 2,000,000 characters. At most 50 signature-validated PNG, JPEG, GIF, or WebP images and 40 MB of image bytes are retained, with a 20 MB per-image limit; unsupported, invalid, omitted, and truncated content remains explicit in the extracted text. Images whose citations fall beyond text truncation are not attached. Extracted document content remains a `text/plain` file attachment with the original document filename, rather than becoming visible user-message text. Supported embedded images become separate image file parts; the extracted text contains `[filename]` citations at the source paragraph, slide object, spreadsheet cell anchor, or OpenDocument text position. Generated image filenames are re-evaluated if the composer changes during asynchronous preparation, avoiding collisions. The store publishes all generated parts atomically only after every data URL is ready. +The composer compares normalized attachment MIME types with the selected model's declared input modalities. It warns when a newly attached file or an existing attachment after a model change requires an unsupported modality, but does not block sending. Missing modality metadata remains unknown and does not produce a warning. + ## Session list rules ### Directory bootstrap scheduling diff --git a/packages/ui/src/sync/attachment-files.test.ts b/packages/ui/src/sync/attachment-files.test.ts index 0e74ce9d..b426ddfc 100644 --- a/packages/ui/src/sync/attachment-files.test.ts +++ b/packages/ui/src/sync/attachment-files.test.ts @@ -2,6 +2,8 @@ import { describe, expect, mock, test } from "bun:test" import { ACCEPTED_ATTACHMENT_EXTENSIONS, ATTACHMENT_ACCEPT, + getAttachmentInputModality, + getUnsupportedAttachmentInputs, prepareAttachmentFile, } from "./attachment-files" @@ -12,6 +14,28 @@ mock.module("heic2any", () => ({ const prepare = (file: File) => Promise.resolve(prepareAttachmentFile(file)) describe("attachment file preparation", () => { + test("maps normalized attachment MIME types to model input modalities", () => { + expect(getAttachmentInputModality("text/plain;charset=utf-8")).toBe("text") + expect(getAttachmentInputModality("image/jpeg")).toBe("image") + expect(getAttachmentInputModality("application/pdf")).toBe("pdf") + expect(getAttachmentInputModality("audio/mpeg")).toBe("audio") + expect(getAttachmentInputModality("video/mp4")).toBe("video") + expect(getAttachmentInputModality("application/octet-stream")).toBe(undefined) + }) + + test("returns only attachment inputs unsupported by the model", () => { + const attachments = [ + { filename: "notes.txt", mimeType: "text/plain" }, + { filename: "photo.jpg", mimeType: "image/jpeg" }, + { filename: "report.pdf", mimeType: "application/pdf" }, + { filename: "unknown.bin", mimeType: "application/octet-stream" }, + ] + + expect(getUnsupportedAttachmentInputs(attachments, ["TEXT", "pdf"])).toEqual([ + { attachment: attachments[1], modality: "image" }, + ]) + }) + test("exposes the expanded code and structured-text formats to pickers", () => { for (const extension of [ "diff", "patch", "ipynb", "jsonl", "ndjson", "har", "svg", "drawio", diff --git a/packages/ui/src/sync/attachment-files.ts b/packages/ui/src/sync/attachment-files.ts index 26d40957..a1e9407f 100644 --- a/packages/ui/src/sync/attachment-files.ts +++ b/packages/ui/src/sync/attachment-files.ts @@ -149,6 +149,33 @@ type OpenCodeAttachmentMimeType = | "application/pdf" | "text/plain" +export type AttachmentInputModality = "text" | "image" | "pdf" | "audio" | "video" + +export const getAttachmentInputModality = (mimeType: string): AttachmentInputModality | undefined => { + const normalizedMimeType = mimeType.toLowerCase().split(";", 1)[0]?.trim() ?? "" + if (normalizedMimeType.startsWith("image/")) return "image" + if (normalizedMimeType.startsWith("audio/")) return "audio" + if (normalizedMimeType.startsWith("video/")) return "video" + if (normalizedMimeType === "application/pdf") return "pdf" + if (normalizedMimeType.startsWith("text/")) return "text" + return undefined +} + +export const getUnsupportedAttachmentInputs = ( + attachments: T[], + supportedInputModalities: string[], +): Array<{ attachment: T; modality: AttachmentInputModality }> => { + const supportedModalities = new Set(supportedInputModalities.map((modality) => modality.toLowerCase())) + const unsupportedInputs: Array<{ attachment: T; modality: AttachmentInputModality }> = [] + for (const attachment of attachments) { + const modality = getAttachmentInputModality(attachment.mimeType) + if (modality && !supportedModalities.has(modality)) { + unsupportedInputs.push({ attachment, modality }) + } + } + return unsupportedInputs +} + const SUPPORTED_BINARY_MIMES = new Map([ ["image/png", "image/png"], ["image/jpeg", "image/jpeg"],