From a4869f980df59c4926270bf708968d85dcc0d291 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 13:26:33 -0400 Subject: [PATCH 1/6] Show what ingestion extracted for V2 chat uploads and match the upload card to generated images A chat upload is stored as a workspace document, so its chat message has no text and /api/get_file_content returned "File content not found" for every workspace-backed upload. The route now returns the linked document's indexed chunks and the stored image analysis, read in the document's own scope, with no storage or screening state in the response. In V2 the uploaded image card is sized to the image like a generated image, its actions are revealed on hover, and Show sources and the file preview show the extraction (engine, AI vision analysis, metadata, indexed text). A cited image shows the cited passage beneath the picture. Refs #1583 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_chat_upload_extraction.py | 154 +++++++++ .../single_app/route_backend_documents.py | 47 +++ .../src/components/chat/ChatFilePreview.tsx | 9 +- .../components/chat/ChatUploadExtraction.tsx | 250 +++++++++++++++ .../src/components/chat/CitationChip.tsx | 1 + .../chat/EnhancedCitationViewer.tsx | 116 ++++++- .../src/components/chat/MessageActions.tsx | 3 +- .../src/components/chat/MessageInspector.tsx | 30 +- .../v2_ui/src/components/chat/MessageList.tsx | 178 +++++++---- application/v2_ui/src/lib/endpoints.ts | 36 ++- ...AT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md | 104 +++++++ docs/reference/chat-controls.md | 6 +- .../test_chat_upload_extraction_details.py | 294 ++++++++++++++++++ ui_tests/test_v2_chat_image_uploads.py | 217 ++++++++++++- 15 files changed, 1356 insertions(+), 91 deletions(-) create mode 100644 application/single_app/functions_chat_upload_extraction.py create mode 100644 application/v2_ui/src/components/chat/ChatUploadExtraction.tsx create mode 100644 docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md create mode 100644 functional_tests/test_chat_upload_extraction_details.py diff --git a/application/single_app/config.py b/application/single_app/config.py index c82b90dd7..2722259e4 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -100,7 +100,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.209" +VERSION = "0.261.210" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_chat_upload_extraction.py b/application/single_app/functions_chat_upload_extraction.py new file mode 100644 index 000000000..672f6232f --- /dev/null +++ b/application/single_app/functions_chat_upload_extraction.py @@ -0,0 +1,154 @@ +# functions_chat_upload_extraction.py +"""Describe what ingestion extracted from the workspace document behind a chat upload. + +A file uploaded into a conversation is stored as a workspace document, so the chat message +itself carries no text. Opening the upload therefore has to read what ingestion produced for +that document: the indexed text that search and chat actually use, and the structured image +analysis that was saved on the document record but was never shown anywhere. + +Everything here is pure so the route can stay thin and the shaping can be tested without +Cosmos DB, AI Search, or Azure credentials. +""" + +WORKSPACE_UPLOAD_FILE_CONTENT_SOURCE = "workspace" +NO_EXTRACTED_CONTENT_MESSAGE = "No extracted content is available for this file yet." +STILL_PROCESSING_MESSAGE = "This file is still being processed. Try again when it finishes." + + +def _clean_text(value): + return value.strip() if isinstance(value, str) else "" + + +def _clean_list(values): + if isinstance(values, str): + values = [values] + if not isinstance(values, (list, tuple)): + return [] + return [value.strip() for value in values if isinstance(value, str) and value.strip()] + + +def summarize_vision_analysis(vision_analysis): + """Return the displayable parts of a stored vision analysis, or None when it has none. + + An analysis that records an error is excluded, because its description is the error + message rather than anything the model saw in the image. + """ + if not isinstance(vision_analysis, dict) or vision_analysis.get("error"): + return None + + summary = { + "model": _clean_text(vision_analysis.get("model") or vision_analysis.get("model_name")), + "description": _clean_text(vision_analysis.get("description")), + "objects": _clean_list(vision_analysis.get("objects")), + "text": _clean_text(vision_analysis.get("text")), + "analysis": _clean_text( + vision_analysis.get("analysis") or vision_analysis.get("contextual_analysis") + ), + } + if not any(summary[key] for key in ("description", "objects", "text", "analysis")): + return None + return summary + + +def render_vision_analysis_text(summary): + """Plain-text rendering for clients that can only show a text body.""" + if not summary: + return "" + + lines = ["AI Vision Analysis"] + if summary.get("model"): + lines.append(f"Model: {summary['model']}") + if summary.get("description"): + lines.append(f"\nDescription: {summary['description']}") + if summary.get("objects"): + lines.append(f"\nObjects Detected: {', '.join(summary['objects'])}") + if summary.get("text"): + lines.append(f"\nVisible Text: {summary['text']}") + if summary.get("analysis"): + lines.append(f"\nContextual Analysis: {summary['analysis']}") + return "\n".join(lines).strip() + + +def join_indexed_chunk_text(chunks): + """Join the indexed text of ordered chunks, skipping empty ones.""" + parts = [] + for chunk in chunks or []: + if not isinstance(chunk, dict): + continue + text = _clean_text(chunk.get("chunk_text")) + if text: + parts.append(text) + return "\n\n".join(parts) + + +def _percentage_complete(document): + try: + return int(document.get("percentage_complete")) + except (TypeError, ValueError): + return None + + +def is_workspace_upload_processing(document): + """Return whether the linked document has not finished ingestion yet.""" + percentage = _percentage_complete(document) + if percentage is not None: + return percentage < 100 + status = _clean_text(document.get("status")).lower() + return bool(status) and not status.startswith("processing complete") + + +def build_workspace_upload_details(document, indexed_chunk_count): + """Return the browser-safe extraction details of the document behind a chat upload. + + Only descriptive ingestion results are returned. Storage locations, scope identifiers and + anything the screening subsystem manages stay on the server. + """ + source = document if isinstance(document, dict) else {} + try: + number_of_pages = int(source.get("number_of_pages")) + except (TypeError, ValueError): + number_of_pages = None + + return { + "document_id": _clean_text(source.get("id")), + "title": _clean_text(source.get("title")), + "abstract": _clean_text(source.get("abstract")), + "keywords": _clean_list(source.get("keywords")), + "status": _clean_text(source.get("status")), + "percentage_complete": _percentage_complete(source), + "number_of_pages": number_of_pages, + "extraction_engine": _clean_text(source.get("extraction_engine")), + "extraction_engine_reason": _clean_text(source.get("extraction_engine_reason")), + "indexed_chunk_count": max(0, int(indexed_chunk_count or 0)), + "vision_analysis": summarize_vision_analysis(source.get("vision_analysis")), + } + + +def build_workspace_upload_file_content(filename, document, chunks): + """Shape the file-content response for a chat upload backed by a workspace document. + + Returns ``(payload, status_code)``. ``file_content`` is always a readable body when the + response succeeds, so a client that only renders text still has something to show: the + indexed text when ingestion saved any, otherwise the stored image analysis. + """ + ordered_chunks = [chunk for chunk in chunks or [] if isinstance(chunk, dict)] + indexed_text = join_indexed_chunk_text(ordered_chunks) + details = build_workspace_upload_details(document, len(ordered_chunks)) + body = indexed_text or render_vision_analysis_text(details["vision_analysis"]) + + if not body: + message = ( + STILL_PROCESSING_MESSAGE + if is_workspace_upload_processing(document if isinstance(document, dict) else {}) + else NO_EXTRACTED_CONTENT_MESSAGE + ) + return {"error": message, "workspace_document": details}, 404 + + return { + "file_content": body, + "filename": filename, + "is_table": False, + "file_content_source": WORKSPACE_UPLOAD_FILE_CONTENT_SOURCE, + "indexed_text_available": bool(indexed_text), + "workspace_document": details, + }, 200 diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 3c476af09..e2ea306a0 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -42,6 +42,7 @@ from utils_cache import invalidate_personal_search_cache from functions_debug import * from functions_activity_logging import log_document_upload, log_document_metadata_update_transaction +from functions_chat_upload_extraction import build_workspace_upload_file_content # Imported explicitly rather than relying on the star imports above: several functions in # this module re-import these names locally, which is a sign the star-imported bindings # cannot be counted on to be the classes rather than the module. @@ -66,6 +67,40 @@ def _extract_citation_document_id(chunk, citation_id): return citation_id +def _workspace_upload_file_content_response(message_item, linked_document, user_id, file_id): + """Return what ingestion extracted for the workspace document behind a chat upload.""" + try: + linked_chunks = get_ordered_document_chunks( + linked_document['id'], + user_id=user_id, + group_id=linked_document.get('group_id'), + public_workspace_id=linked_document.get('public_workspace_id'), + ) + except ScreeningError as screening_error: + return jsonify({ + 'error': screening_error.public_message, + 'error_code': screening_error.code, + }), screening_error.status_code + except (LookupError, PermissionError): + return jsonify({'error': 'File not found in conversation'}), 404 + except Exception as chunk_error: + log_event( + "[GET_FILE_CONTENT] Failed to read the indexed content of a workspace-backed upload.", + extra={ + 'file_id': file_id, + 'document_id': linked_document.get('id'), + 'error_type': type(chunk_error).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + return jsonify({'error': 'Unable to load the extracted content for this file.'}), 500 + + filename = linked_document.get('file_name') or message_item.get('filename') or 'Untitled' + payload, status_code = build_workspace_upload_file_content(filename, linked_document, linked_chunks) + return jsonify(payload), status_code + + def _normalize_citation_lookup_value(value): if value is None: return '' @@ -555,6 +590,18 @@ def get_file_content(): "blob_path": linked_path, }] + # A workspace-backed upload keeps no text on the chat message; its content is + # whatever ingestion indexed for the linked document. Tables keep their own path. + if ( + linked_document is not None + and SCREENING_FIELD not in linked_document + and not items[0].get('is_table') + and not items[0].get('file_content') + ): + return _workspace_upload_file_content_response( + items[0], linked_document, user_id, file_id, + ) + debug_print(f"[GET_FILE_CONTENT] Found {len(items)} items for file_id={file_id}") debug_print(f"[GET_FILE_CONTENT] First item structure: {json.dumps(items[0], default=str, indent=2)}") add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="File found, processing content: " + str(items)) diff --git a/application/v2_ui/src/components/chat/ChatFilePreview.tsx b/application/v2_ui/src/components/chat/ChatFilePreview.tsx index 6adbfc2f7..6a6d0934a 100644 --- a/application/v2_ui/src/components/chat/ChatFilePreview.tsx +++ b/application/v2_ui/src/components/chat/ChatFilePreview.tsx @@ -10,6 +10,10 @@ // citations keep the upload in blob storage; without them only the extracted text survives, // and offering a download that produces a text approximation of a spreadsheet would be worse // than offering nothing. +// +// An upload stored as a workspace document has no text on its message. What it contains is +// what ingestion extracted, so that is what is shown: the indexed text and, for an image, the +// vision analysis. import { useEffect, useState } from 'react'; import { createPortal } from 'react-dom'; @@ -21,6 +25,7 @@ import { } from '../../lib/endpoints'; import { parseCsvPreview } from '../../lib/csvPreview'; import { GlassButton, GlassPanel } from '../ui/primitives'; +import { ChatUploadExtraction } from './ChatUploadExtraction'; function CsvTable({ content }: { content: string }) { const preview = parseCsvPreview(content); @@ -170,9 +175,7 @@ export function ChatFilePreview({ ) : data.is_table ? ( ) : ( -
-                            {String(data.file_content ?? '')}
-                        
+ )} diff --git a/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx b/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx new file mode 100644 index 000000000..c34f0509e --- /dev/null +++ b/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx @@ -0,0 +1,250 @@ +// ChatUploadExtraction.tsx +// What ingestion extracted from a file uploaded into a conversation. +// +// A chat upload is stored as a workspace document, so the useful answer to "what is in this +// file" is what ingestion produced for it: the text that search and chat actually read and, +// for an image, the analysis the vision model wrote. It is the same material a cited passage +// comes from, so the upload's preview and its Sources panel both show it the same way. + +import { useEffect, useState } from 'react'; +import { clsx } from 'clsx'; +import { Eye, Loader2, ScanText, TriangleAlert } from 'lucide-react'; +import { + fetchChatFileContent, + type ChatFileContent, + type WorkspaceUploadVisionAnalysis, +} from '../../lib/endpoints'; + +const ENGINE_LABELS: Record = { + content_understanding: 'Azure AI Content Understanding', + document_intelligence: 'Azure AI Document Intelligence', +}; + +/** A readable name for a stored extraction engine id, or '' when none was recorded. */ +export function describeExtractionEngine(engine?: string): string { + const normalized = String(engine ?? '').trim().toLowerCase(); + if (!normalized) { + return ''; + } + return ENGINE_LABELS[normalized] ?? normalized.replace(/_/g, ' '); +} + +function Field({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+

+ {label} +

+ {children} +
+ ); +} + +function VisionAnalysisBlock({ analysis }: { analysis: WorkspaceUploadVisionAnalysis }) { + const objects = (analysis.objects ?? []).filter((item) => String(item).trim()); + + return ( +
+

+ + AI vision analysis + {analysis.model ? ( + · {analysis.model} + ) : null} +

+ {analysis.description ? ( + +

+ {analysis.description} +

+
+ ) : null} + {objects.length > 0 ? ( + +
    + {objects.map((item, index) => ( +
  • + {item} +
  • + ))} +
+
+ ) : null} + {analysis.text ? ( + +

{analysis.text}

+
+ ) : null} + {analysis.analysis ? ( + +

+ {analysis.analysis} +

+
+ ) : null} +
+ ); +} + +/** + * The extraction results for one upload. + * + * Content that did not come from a workspace document -- an older upload whose text lives on + * the message -- has no extraction details, so it is shown as the plain text it always was. + */ +export function ChatUploadExtraction({ data }: { data: ChatFileContent }) { + const text = String(data.file_content ?? ''); + const details = data.workspace_document; + + if (!details) { + return
{text}
; + } + + const engine = describeExtractionEngine(details.extraction_engine); + const vision = details.vision_analysis ?? null; + const keywords = (details.keywords ?? []).filter((item) => String(item).trim()); + const indexed = data.indexed_text_available === true; + + return ( +
+ {engine || details.extraction_engine_reason ? ( +

+ + + {engine ? ( + <> + Extracted with {engine} + {details.extraction_engine_reason ? '. ' : ''} + + ) : null} + {details.extraction_engine_reason} + +

+ ) : null} + + {details.title || details.abstract || keywords.length > 0 ? ( +
+ {details.title ? ( + +

{details.title}

+
+ ) : null} + {details.abstract ? ( + +

+ {details.abstract} +

+
+ ) : null} + {keywords.length > 0 ? ( + +

{keywords.join(', ')}

+
+ ) : null} +
+ ) : null} + + {vision ? : null} + + {indexed ? ( + // Collapsed when the vision analysis already says the same thing in a readable + // form; open when it is the only account of what the file contains. +
+ + Indexed text + + (what search and chat read + {details.indexed_chunk_count + ? `, ${details.indexed_chunk_count} chunk${details.indexed_chunk_count === 1 ? '' : 's'}` + : ''} + ) + + +
+                        {text}
+                    
+
+ ) : ( +

+ + {typeof details.percentage_complete === 'number' && + details.percentage_complete < 100 + ? 'Still processing. The indexed text appears here once ingestion finishes.' + : 'Nothing from this file was indexed for search, so chat cannot cite it. Reprocess or re-upload the file to index it.'} +

+ )} +
+ ); +} + +/** + * Loads and shows the extraction results for one upload. + * + * Kept separate from the modal preview so the message inspector can show the same content + * inline, without a second copy of the loading and failure handling. + */ +export function ChatUploadExtractionLoader({ + conversationId, + fileId, + className, +}: { + conversationId: string; + fileId: string; + className?: string; +}) { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + let cancelled = false; + setData(null); + setError(null); + + fetchChatFileContent(conversationId, fileId) + .then((result) => { + if (cancelled) { + return; + } + if (result.error) { + setError(result.error); + return; + } + setData(result); + }) + .catch((cause: unknown) => { + if (!cancelled) { + setError( + cause instanceof Error ? cause.message : 'Could not load the file.', + ); + } + }); + + return () => { + cancelled = true; + }; + }, [conversationId, fileId]); + + return ( +
+ {error ? ( +

+ + {error} +

+ ) : !data ? ( +

+ + Loading what was extracted… +

+ ) : ( + + )} +
+ ); +} diff --git a/application/v2_ui/src/components/chat/CitationChip.tsx b/application/v2_ui/src/components/chat/CitationChip.tsx index 98cd50105..e3ccb3ce9 100644 --- a/application/v2_ui/src/components/chat/CitationChip.tsx +++ b/application/v2_ui/src/components/chat/CitationChip.tsx @@ -336,6 +336,7 @@ function CitationTarget({ metadata={enhanced.metadata} onClose={onClose} onFallback={onFallback} + citationId={citation.citationId} /> ); } diff --git a/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx b/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx index a84b4adbf..d5c09e348 100644 --- a/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx +++ b/application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx @@ -22,6 +22,7 @@ import { enhancedCitationImageUrl, enhancedCitationMediaUrl, enhancedCitationVisioUrl, + fetchCitation, fetchEnhancedCitationPdf, fetchTabularPreview, tabularWorkspaceDownloadUrl, @@ -43,6 +44,8 @@ interface ViewerProps { metadata: EnhancedCitationMetadata; /** Called when this viewer cannot render, so the caller can show the text passage. */ onFail: (reason: string) => void; + /** The citation being viewed, when known, so a viewer can show the cited passage too. */ + citationId?: string; } function ViewerError({ message }: { message: string }) { @@ -155,25 +158,102 @@ function PdfViewer({ docId, location, onFail }: ViewerProps) { /* Image */ /* -------------------------------------------------------------------------- */ -function ImageViewer({ docId, metadata, onFail }: ViewerProps) { +/** + * What ingestion extracted from a cited image. + * + * An image has no page to scroll to, so the picture alone cannot show why it was cited. The + * cited passage is the text the assistant actually read -- any recognised text plus the + * image's description -- and is shown beneath the picture rather than instead of it. + * Nothing is drawn when the passage cannot be loaded; the image is still the source. + */ +function CitedImagePassage({ + citationId, + docId, + location, +}: { + citationId: string; + docId: string; + location: string; +}) { + const [passage, setPassage] = useState(null); + const [failed, setFailed] = useState(false); + + useEffect(() => { + let cancelled = false; + setPassage(null); + setFailed(false); + + fetchCitation({ + citation_id: citationId, + document_id: docId || undefined, + page_number: location || undefined, + }) + .then((result) => { + if (!cancelled) { + setPassage(String(result?.cited_text ?? '').trim()); + } + }) + .catch(() => { + if (!cancelled) { + setFailed(true); + } + }); + + return () => { + cancelled = true; + }; + }, [citationId, docId, location]); + + if (failed || passage === '') { + return null; + } + + return ( +
+ + What was extracted from this image + + {passage === null ? ( +

+ + Loading the cited passage… +

+ ) : ( +

+ {passage} +

+ )} +
+ ); +} + +function ImageViewer({ docId, location, metadata, onFail, citationId }: ViewerProps) { const [loading, setLoading] = useState(true); return ( -
- {loading && } - {metadata.file_name setLoading(false)} - onError={() => { - setLoading(false); - onFail('The image could not be loaded.'); - }} - className={clsx( - 'max-h-full max-w-full rounded-xl object-contain', - loading && 'hidden', - )} - /> +
+
+ {loading && } + {metadata.file_name setLoading(false)} + onError={() => { + setLoading(false); + onFail('The image could not be loaded.'); + }} + className={clsx( + 'max-h-full max-w-full rounded-xl object-contain', + loading && 'hidden', + )} + /> +
+ {citationId ? ( + + ) : null}
); } @@ -426,6 +506,7 @@ export function EnhancedCitationViewer({ metadata, onClose, onFallback, + citationId, }: { type: EnhancedCitationType; docId: string; @@ -434,6 +515,8 @@ export function EnhancedCitationViewer({ metadata: EnhancedCitationMetadata; onClose: () => void; onFallback: (reason: string) => void; + /** The citation being viewed; lets the image viewer show the cited passage. */ + citationId?: string; }) { useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { @@ -453,6 +536,7 @@ export function EnhancedCitationViewer({ location, metadata, onFail: handleFail, + citationId, }; // Tabular has its own download endpoint; everything else uses the generic one. diff --git a/application/v2_ui/src/components/chat/MessageActions.tsx b/application/v2_ui/src/components/chat/MessageActions.tsx index d2dbd7f73..65efa0f2c 100644 --- a/application/v2_ui/src/components/chat/MessageActions.tsx +++ b/application/v2_ui/src/components/chat/MessageActions.tsx @@ -63,7 +63,8 @@ import { type PromptDraft, } from '../prompts/PromptEditorDialog'; -function IconButton({ +/** A compact icon control, shared so message rows of every kind look and behave alike. */ +export function IconButton({ label, onClick, children, diff --git a/application/v2_ui/src/components/chat/MessageInspector.tsx b/application/v2_ui/src/components/chat/MessageInspector.tsx index ad1fd6f16..f5598db7d 100644 --- a/application/v2_ui/src/components/chat/MessageInspector.tsx +++ b/application/v2_ui/src/components/chat/MessageInspector.tsx @@ -29,6 +29,7 @@ import type { ChatMessage, Json, PersistedThought } from '../../lib/types'; import { buildToolResultView, type RowMode } from '../../lib/agentCitationRows'; import { GlassButton } from '../ui/primitives'; import { normalizePersistedThought, ThoughtsList } from './ThoughtsList'; +import { ChatUploadExtractionLoader } from './ChatUploadExtraction'; export type InspectorSection = 'details' | 'sources' | 'reasoning'; @@ -356,13 +357,19 @@ export function MessageInspector({ onClose: () => void; }) { const sources = readSources(message as unknown as Json); + // An uploaded file cites nothing and has no reasoning. Its sources are what ingestion + // extracted from it, which is the material any later citation of the file comes from. + const upload = message.role === 'file'; + const conversationId = String(message.conversation_id ?? '').trim(); + const fileId = String(message.id ?? '').trim(); // Reasoning and sources only exist for a generated response. - const available: InspectorSection[] = - message.role === 'user' - ? ['details'] - : ['details', 'sources', 'reasoning']; - const active = available.includes(section) ? section : 'details'; + const available: InspectorSection[] = upload + ? ['sources', 'details'] + : message.role === 'user' + ? ['details'] + : ['details', 'sources', 'reasoning']; + const active = available.includes(section) ? section : available[0]; return (
@@ -398,7 +405,18 @@ export function MessageInspector({
{active === 'details' && } - {active === 'sources' && } + {active === 'sources' && upload && ( + conversationId && fileId ? ( + + ) : ( + This upload has no stored extraction to show. + ) + )} + {active === 'sources' && !upload && } {active === 'reasoning' && }
diff --git a/application/v2_ui/src/components/chat/MessageList.tsx b/application/v2_ui/src/components/chat/MessageList.tsx index 0db44e6a9..1b2ec2449 100644 --- a/application/v2_ui/src/components/chat/MessageList.tsx +++ b/application/v2_ui/src/components/chat/MessageList.tsx @@ -5,12 +5,14 @@ import { ReasoningAdjustmentNotice } from './ReasoningAdjustmentNotice'; import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { clsx } from 'clsx'; import { + BookOpen, Brain, ChevronDown, EyeOff, FileText, Image as ImageIcon, ImageOff, + Info, PenLine, RefreshCw, Reply, @@ -29,7 +31,7 @@ import { AssistantMarkdown } from './AssistantMarkdown'; import { ChatFilePreview } from './ChatFilePreview'; import { GeneratedArtifactCard } from './GeneratedArtifactCard'; import { AnalysisResult } from './AnalysisResult'; -import { MessageActions } from './MessageActions'; +import { IconButton, MessageActions } from './MessageActions'; import { OrchestrationMessageRecovery } from './OrchestrationRecoveryNotice'; import { OrchestrationOutputs } from './OrchestrationOutputs'; import { WorkflowProposalCards } from './WorkflowProposalCard'; @@ -365,7 +367,13 @@ export interface UploadedImageFileCardProps { localPreviewKeys?: readonly unknown[]; onOpenFile?: () => void; onUseAsReference?: () => void; + /** + * Controls shown in the row below the image, revealed on hover or keyboard focus the same + * way a generated image's actions are. Only drawn while the image itself is shown. + */ actions?: ReactNode; + /** Keeps the action row visible, for instance while a panel it opened is showing. */ + actionsPinned?: boolean; } export function UploadedImageFileCard({ @@ -378,6 +386,7 @@ export function UploadedImageFileCard({ onOpenFile, onUseAsReference, actions, + actionsPinned = false, }: UploadedImageFileCardProps) { const [lightboxOpen, setLightboxOpen] = useState(false); const [decodeFailed, setDecodeFailed] = useState(false); @@ -410,51 +419,59 @@ export function UploadedImageFileCard({ ); } + // Sized to the image rather than to the message column, so an upload and a generated + // image sit in the thread the same way instead of the upload being letterboxed. return ( -
-
- + {/* Zero intrinsic width, so a long name is truncated to the image rather + than stretching the card past it. */} +

+ {fileName} +

+
+
+ {actions ? ( +
- {visibleUrl ? ( - {fileName} setDecodeFailed(true)} - className="max-h-80 max-w-full rounded-xl object-contain" - /> - ) : ( -
- -

- {preview.status === 'processing' ? 'Processing image…' : 'Loading image…'} -

-
- )} - -
-

{fileName}

- {onOpenFile ? ( - - ) : null} + {actions}
- {actions ?
{actions}
: null} -
+ ) : null} {preview.status === 'processing' && !visibleUrl ? (

Processing image…

) : null} @@ -467,7 +484,7 @@ export function UploadedImageFileCard({ onUseAsReference={onUseAsReference} /> ) : null} -
+ ); } @@ -563,33 +580,70 @@ function FileMessage({ message }: { message: ChatMessage }) { && isReferenceImageFileName(fileName) && !isHeifFileName(fileName); const imageReference: ImageReferenceRequest = { type: 'message', message_id: fileId }; const uploadRevisions = useImageRevisions(fileId, '', buildChatImagePreviewUrl(fileId, 'display')); - const uploadActions = referenceCapable ? ( + const [inspector, setInspector] = useState(null); + /** Toggle a section, closing the panel when the open one is clicked again. */ + const inspect = (section: InspectorSection) => + setInspector((current) => (current === section ? null : section)); + // The same row a generated image has: how the upload was read first, then what can be + // done with it. Sources opens what ingestion extracted, which is what any later citation + // of this file is drawn from. + const uploadActions = ( <> - - {capability.editing && ( + {openable && ( +
+ inspect('sources')} + > + + + inspect('details')} + > + + + setPreviewOpen(true)} + > + + +
+ )} + {referenceCapable && ( + + )} + {referenceCapable && capability.editing && ( )} - ) : null; + ); + const hasUploadActions = openable || referenceCapable; return (
{author &&

{author}

} {imagePreviewable ? ( @@ -602,7 +656,8 @@ function FileMessage({ message }: { message: ChatMessage }) { localPreviewKeys={[fileId, workspaceDocumentId]} onOpenFile={openable ? () => setPreviewOpen(true) : undefined} onUseAsReference={referenceCapable ? () => addComposerImageReference(imageReference) : undefined} - actions={uploadActions} + actions={hasUploadActions ? uploadActions : null} + actionsPinned={inspector !== null} /> ) : ( )} + {inspector && imagePreviewable && openable && ( + setInspector(null)} + /> + )} + {orchestrationFile && (

diff --git a/application/v2_ui/src/lib/endpoints.ts b/application/v2_ui/src/lib/endpoints.ts index bf1e59836..3764e7418 100644 --- a/application/v2_ui/src/lib/endpoints.ts +++ b/application/v2_ui/src/lib/endpoints.ts @@ -1175,6 +1175,36 @@ export function chatUploadTabularDownloadUrl(conversationId: string, fileId: str /* Chat file uploads */ /* -------------------------------------------------------------------------- */ +/** The displayable parts of the vision analysis stored on an uploaded image's document. */ +export interface WorkspaceUploadVisionAnalysis { + model?: string; + description?: string; + objects?: string[]; + text?: string; + analysis?: string; +} + +/** + * What ingestion recorded for the workspace document behind a chat upload. + * + * A chat upload is stored as a workspace document, so these are the extraction results the + * assistant actually works from rather than anything kept on the chat message itself. + */ +export interface WorkspaceUploadDetails { + document_id?: string; + title?: string; + abstract?: string; + keywords?: string[]; + status?: string; + percentage_complete?: number | null; + number_of_pages?: number | null; + /** `content_understanding` or `document_intelligence`. */ + extraction_engine?: string; + extraction_engine_reason?: string; + indexed_chunk_count?: number; + vision_analysis?: WorkspaceUploadVisionAnalysis | null; +} + /** What `/api/get_file_content` returns for a file uploaded straight into a conversation. */ export interface ChatFileContent { file_content?: string; @@ -1183,9 +1213,13 @@ export interface ChatFileContent { is_table?: boolean; /** * Where the content came from. `blob` means the original file is still in storage and - * can be downloaded; anything else means only the extracted text survives. + * can be downloaded; `workspace` means it is what ingestion indexed for the linked + * workspace document; anything else means only the extracted text survives. */ file_content_source?: string; + /** For a workspace-backed upload: whether `file_content` is indexed text. */ + indexed_text_available?: boolean; + workspace_document?: WorkspaceUploadDetails; error?: string; } diff --git a/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md b/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md new file mode 100644 index 000000000..d9a1c14b4 --- /dev/null +++ b/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md @@ -0,0 +1,104 @@ +# V2 Chat Upload Extraction and Image Card Parity Fix + +Fixed in version: **0.261.210** + +Related issue: [#1583](https://github.com/microsoft/simplechat/issues/1583) + +## Issue Description + +Three problems made it hard to see what SimpleChat knew about an image uploaded into a V2 +conversation. + +- **File content not found.** Opening an uploaded image, or any other file uploaded into a + conversation with the personal workspace enabled, showed "File content not found". The + classic interface showed the extracted text and the AI vision analysis. +- **Cited images showed only the picture.** A citation of an image opened the image, with no + sign of the text the assistant was given or the image's description. +- **The upload card did not match a generated image.** The uploaded image filled the whole + message column, with the picture letterboxed in a grey box. Its actions were always visible + and could not show sources, while a generated image in the same thread was drawn at its own + size with its actions revealed on hover. + +## Root Cause + +A chat upload is stored as a workspace document and processed by the normal ingestion +pipeline. The chat message only records `workspace_document_id` and +`file_content_source: "workspace"`. It has no `file_content`. + +`/api/get_file_content` only combined `file_content` from the message rows, or read the blob +for a screened document. Every unscreened workspace-backed upload therefore fell through to its +empty-content 404. The document's indexed chunks and its stored `vision_analysis` were never +read. + +The V2 image citation viewer rendered only `/api/enhanced_citations/image`. It never requested +the cited passage. + +`UploadedImageFileCard` was laid out as a full-width attachment card rather than the compact +card `ImageMessage` uses for generated images. + +## Technical Details + +### Files Modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_chat_upload_extraction.py` | New pure helpers that turn the linked document and its ordered chunks into the response: the indexed text, the stored vision analysis with error analyses dropped, the engine and its reason, and title, summary and keywords. Storage, scope and screening fields are never included. | +| `application/single_app/route_backend_documents.py` | For an unscreened, non-table upload linked to a workspace document, `get_file_content` reads the document's chunks with `get_ordered_document_chunks` in the document's own scope and returns the helper's payload. Screening errors keep their public wording, a missing document is a 404, and any other failure returns a fixed message. | +| `application/v2_ui/src/components/chat/ChatUploadExtraction.tsx` | New component that renders the extraction: the engine, the AI vision analysis (description, objects, visible text, contextual analysis), metadata, and the indexed text. It warns when nothing was indexed. | +| `application/v2_ui/src/components/chat/ChatFilePreview.tsx` | **Uploaded file** renders the extraction instead of a bare text block. | +| `application/v2_ui/src/components/chat/MessageInspector.tsx` | An uploaded file message gets **Sources**, which shows its extraction, and **Details**. | +| `application/v2_ui/src/components/chat/MessageList.tsx` | The uploaded image card is sized to the image like a generated image. Its actions row (Show sources, Message details, Open file preview, Use as reference, Edit) is revealed on hover or focus and stays visible while its panel is open. | +| `application/v2_ui/src/components/chat/MessageActions.tsx` | `IconButton` is exported so the upload card uses the same controls as other messages. | +| `application/v2_ui/src/components/chat/EnhancedCitationViewer.tsx`, `CitationChip.tsx` | The image viewer shows **What was extracted from this image**, the cited passage, beneath the image. | +| `application/v2_ui/src/lib/endpoints.ts` | Types for the new `workspace_document` details. | + +### Response Shape + +For a workspace-backed upload, `/api/get_file_content` returns: + +```json +{ + "file_content": "indexed text, or the vision analysis when nothing was indexed", + "filename": "lab.png", + "is_table": false, + "file_content_source": "workspace", + "indexed_text_available": true, + "workspace_document": { + "document_id": "…", + "title": "…", + "status": "Processing complete - final metadata extracted", + "percentage_complete": 100, + "extraction_engine": "document_intelligence", + "extraction_engine_reason": "…", + "indexed_chunk_count": 1, + "vision_analysis": {"model": "gpt-5.4", "description": "…", "objects": ["…"], "text": "", "analysis": "…"} + } +} +``` + +`file_content` is always readable text, so the classic file popup, which renders only +`file_content`, also works for these uploads now. When there is nothing to show, the route +returns 404 with "This file is still being processed. Try again when it finishes." while +ingestion is running, or "No extracted content is available for this file yet." afterwards. + +Tabular uploads and screened documents keep their existing paths. + +## Testing + +- `functional_tests/test_chat_upload_extraction_details.py`: response shaping, fallback to the + stored vision analysis, processing and absent wording, the route helper's scope and error + handling, and the position of the new branch before the empty-content 404. +- `ui_tests/test_v2_chat_image_uploads.py`: the card is sized to a 320×200 image, its actions + appear only on hover, **Show sources** and **Uploaded file** show the vision analysis and + indexed text, a processing file explains itself, and a cited image shows the extracted + passage. +- Route policy tests in `functional_tests/route_tests/` pass unchanged. No route was added or + moved. + +## Validation + +| Before | After | +| --- | --- | +| **Uploaded file** said "File content not found" for every workspace-backed upload. | It shows how the file was read, the AI vision analysis, and the indexed text. | +| A cited image showed only the picture. | The cited passage appears beneath the image. | +| The upload filled the message column, letterboxed, with its actions always visible. | The upload is drawn at the image's size, and its actions, including **Show sources**, appear on hover, as a generated image's do. | diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md index f704540ca..05646ee10 100644 --- a/docs/reference/chat-controls.md +++ b/docs/reference/chat-controls.md @@ -152,7 +152,11 @@ not part of the generated inventory above. See | Control | What it does | Why you would use it | Enabled by | | --- | --- | --- | --- | -| Uploaded image card | Draws an uploaded PNG, JPG, BMP, or TIFF image in the thread, shows **Processing image…** until the workspace is ready, and opens the full-size viewer when selected. HEIC is drawn only in Safari. | See what was asked about instead of a file name, and check that the right picture was attached. | [`enable_chat_file_uploads`]({{ '/admin/workspaces/' | relative_url }})
[`enable_user_workspace`]({{ '/admin/workspaces/' | relative_url }}) | +| Uploaded image card | Draws an uploaded PNG, JPG, BMP, or TIFF image in the thread at the image's own size, the same way a generated image is drawn, shows **Processing image…** until the workspace is ready, and opens the full-size viewer when selected. HEIC is drawn only in Safari. Since **0.261.210**, its actions appear when you point at or tab to the image, as a generated image's do. | See what was asked about instead of a file name, and check that the right picture was attached. | [`enable_chat_file_uploads`]({{ '/admin/workspaces/' | relative_url }})
[`enable_user_workspace`]({{ '/admin/workspaces/' | relative_url }}) | +| Show sources (uploaded image) | Since **0.261.210**, opens a panel under the upload showing what ingestion extracted from it: the engine that read it and why, the AI vision analysis (description, objects, visible text, contextual analysis), the title and summary from metadata extraction, and the indexed text that search and chat read. It warns when nothing from the file was indexed. | Check what the assistant can actually know about a picture before relying on its answers, or find out why an image is not being cited. | Same as Uploaded image card | +| Message details (uploaded image) | Since **0.261.210**, shows the diagnostics recorded for the upload message in the same panel. | Confirm which conversation and workspace the upload belongs to. | Same as Uploaded image card | +| Open file preview | Opens **Uploaded file** with the same extraction results as **Show sources**. Before **0.261.210**, a workspace-backed upload showed **File content not found** here. | Read the extraction in a larger view. | Same as Uploaded image card | +| Cited image | Since **0.261.210**, a citation of an image opens the image with **What was extracted from this image** beneath it: the passage the assistant was given, including any recognized text and the image's description. | See why an image was cited, since a picture has no page to point to. | [`enable_enhanced_citations`]({{ '/admin/chat/' | relative_url }}) | | Attach a reference image | With **Image** on, uploads, pastes, or drops a PNG, JPG, BMP, or TIFF picture that is sent to the image model with your prompt. HEIC is refused with a conversion hint. | Base a new image on a real picture: your house, your face, or a map. | [`enable_image_generation`]({{ '/admin/ai-models/' | relative_url }}) and an image model that can use references | | Documents (in Image mode) | Lists only workspace images, most recent first, and adds the ones you pick as references. Search by name to reach an older image. | Reuse a picture already stored in a workspace without uploading it again. | Same as Attach a reference image, plus a workspace | | Reference count | Shows how many references are attached and how many the image model accepts, for example **2 / 10 reference images**, or explains why references are unavailable. | Stay within the model's limit before sending. | Same as Attach a reference image | diff --git a/functional_tests/test_chat_upload_extraction_details.py b/functional_tests/test_chat_upload_extraction_details.py new file mode 100644 index 000000000..4fd7d1f21 --- /dev/null +++ b/functional_tests/test_chat_upload_extraction_details.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +# test_chat_upload_extraction_details.py +""" +Functional test for chat upload extraction details. +Version: 0.261.210 +Implemented in: 0.261.210 + +This test ensures a chat upload backed by a workspace document returns what ingestion +extracted for it (the indexed text and the stored image analysis) instead of +"File content not found", and that the response never carries storage or screening state. +""" + +import ast +import logging +import sys +from pathlib import Path + +from flask import Flask, jsonify + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_DIR = REPO_ROOT / "application" / "single_app" +ROUTE_FILE = APP_DIR / "route_backend_documents.py" + +sys.path.insert(0, str(APP_DIR)) +sys.path.insert(0, str(REPO_ROOT / "functional_tests")) + +import functions_chat_upload_extraction as extraction # noqa: E402 +from test_support.versioning import assert_app_version_at_least # noqa: E402 + +VISION_ANALYSIS = { + "model": "gpt-5.4", + "description": "A grayscale image of a large indoor lab with a mostly empty tiled floor.", + "objects": ["checkerboard calibration panels", " ", "equipment rack"], + "text": "", + "analysis": "Likely a robotics or computer vision test space.", +} + +LINKED_DOCUMENT = { + "id": "doc-lab", + "file_name": "lab.png", + "title": "AI Vision Analysis of Indoor Environment", + "abstract": "An indoor lab.", + "keywords": ["lab", "calibration"], + "status": "Processing complete - final metadata extracted", + "percentage_complete": 100, + "number_of_pages": 1, + "extraction_engine": "document_intelligence", + "extraction_engine_reason": "Content Understanding returned no content, so Document Intelligence Layout was used", + "vision_analysis": VISION_ANALYSIS, + # Server-only state that must never reach the browser through this response. + "blob_container": "user-documents", + "blob_path": "user/lab.png", + "user_id": "user-1", + "content_screening": {"state": "released"}, +} + + +def test_summarize_vision_analysis_keeps_only_displayable_parts(): + """Errors and empty analyses are dropped; aliases and lists are normalized.""" + summary = extraction.summarize_vision_analysis(VISION_ANALYSIS) + assert summary == { + "model": "gpt-5.4", + "description": VISION_ANALYSIS["description"], + "objects": ["checkerboard calibration panels", "equipment rack"], + "text": "", + "analysis": VISION_ANALYSIS["analysis"], + }, summary + + assert extraction.summarize_vision_analysis(None) is None + assert extraction.summarize_vision_analysis("text") is None + assert extraction.summarize_vision_analysis({ + "description": "Model returned empty content with no refusal message", + "error": "Model returned empty content with no refusal message", + "parse_failed": True, + }) is None + assert extraction.summarize_vision_analysis({"model": "gpt-5.4", "objects": []}) is None + + aliased = extraction.summarize_vision_analysis({ + "model_name": "gpt-4o", + "objects": "chair", + "contextual_analysis": "An office.", + }) + assert aliased["model"] == "gpt-4o" + assert aliased["objects"] == ["chair"] + assert aliased["analysis"] == "An office." + + +def test_indexed_upload_returns_joined_chunks_and_details(): + """Indexed chunks are the body; details describe how the file was read.""" + chunks = [ + {"chunk_text": "\n\n=== AI Vision Analysis ===\nModel: gpt-5.4"}, + {"chunk_text": " "}, + "not-a-chunk", + {"chunk_text": "Second page"}, + ] + payload, status = extraction.build_workspace_upload_file_content("lab.png", LINKED_DOCUMENT, chunks) + + assert status == 200, payload + assert payload["file_content"].startswith("") + assert payload["file_content"].endswith("Second page") + assert payload["filename"] == "lab.png" + assert payload["is_table"] is False + assert payload["file_content_source"] == "workspace" + assert payload["indexed_text_available"] is True + + details = payload["workspace_document"] + assert details["document_id"] == "doc-lab" + assert details["extraction_engine"] == "document_intelligence" + assert details["extraction_engine_reason"].startswith("Content Understanding returned no content") + assert details["indexed_chunk_count"] == 3 + assert details["keywords"] == ["lab", "calibration"] + assert details["vision_analysis"]["description"] == VISION_ANALYSIS["description"] + + exposed = set(details) + for server_only in ("blob_container", "blob_path", "user_id", "content_screening", "id"): + assert server_only not in exposed, f"{server_only} leaked into the response" + + +def test_unindexed_image_falls_back_to_the_stored_vision_analysis(): + """An image ingestion never indexed still shows what the vision model saw.""" + document = {**LINKED_DOCUMENT, "status": "Processing complete - no text found in image", "title": ""} + payload, status = extraction.build_workspace_upload_file_content("22222.png", document, []) + + assert status == 200, payload + assert payload["indexed_text_available"] is False + assert payload["workspace_document"]["indexed_chunk_count"] == 0 + assert "AI Vision Analysis" in payload["file_content"] + assert VISION_ANALYSIS["description"] in payload["file_content"] + assert "Objects Detected: checkerboard calibration panels, equipment rack" in payload["file_content"] + + +def test_nothing_to_show_explains_processing_or_absence(): + """No chunks and no analysis is a clear 404, worded by whether ingestion finished.""" + processing = {"id": "doc-new", "status": "Sending lab.png to Azure AI Content Understanding...", "percentage_complete": 35} + payload, status = extraction.build_workspace_upload_file_content("lab.png", processing, []) + assert status == 404 + assert payload["error"] == extraction.STILL_PROCESSING_MESSAGE + assert payload["workspace_document"]["percentage_complete"] == 35 + + finished = {"id": "doc-done", "status": "Processing complete - no text found in image", "percentage_complete": 100} + payload, status = extraction.build_workspace_upload_file_content("lab.png", finished, []) + assert status == 404 + assert payload["error"] == extraction.NO_EXTRACTED_CONTENT_MESSAGE + + status_only = {"id": "doc-queued", "status": "Queued for processing"} + payload, status = extraction.build_workspace_upload_file_content("lab.png", status_only, None) + assert status == 404 + assert payload["error"] == extraction.STILL_PROCESSING_MESSAGE + + +def _load_route_helper(): + """Compile the route's helper on its own, with its collaborators supplied by the test.""" + tree = ast.parse(ROUTE_FILE.read_text(encoding="utf-8-sig")) + helper = next( + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "_workspace_upload_file_content_response" + ) + module = ast.Module(body=[helper], type_ignores=[]) + + class ScreeningError(Exception): + def __init__(self, public_message, code, status_code): + super().__init__(public_message) + self.public_message = public_message + self.code = code + self.status_code = status_code + + logged = [] + namespace = { + "jsonify": jsonify, + "logging": logging, + "ScreeningError": ScreeningError, + "build_workspace_upload_file_content": extraction.build_workspace_upload_file_content, + "log_event": lambda message, **kwargs: logged.append((message, kwargs)), + "get_ordered_document_chunks": None, + } + exec(compile(module, str(ROUTE_FILE), "exec"), namespace) + return namespace, ScreeningError, logged + + +def test_route_helper_reads_chunks_in_the_linked_documents_scope(): + """The helper reads chunks with the stored document's own scope and returns them.""" + namespace, _screening_error, _logged = _load_route_helper() + calls = [] + + def fake_chunks(document_id, user_id=None, group_id=None, public_workspace_id=None): + calls.append((document_id, user_id, group_id, public_workspace_id)) + return [{"chunk_text": "Indexed image description"}] + + namespace["get_ordered_document_chunks"] = fake_chunks + group_document = {**LINKED_DOCUMENT, "group_id": "group-1"} + app = Flask(__name__) + with app.app_context(): + response, status = namespace["_workspace_upload_file_content_response"]( + {"filename": "fallback.png"}, group_document, "user-1", "file-1", + ) + assert status == 200 + assert response.get_json()["file_content"] == "Indexed image description" + assert response.get_json()["filename"] == "lab.png" + assert calls == [("doc-lab", "user-1", "group-1", None)], calls + + +def test_route_helper_never_returns_raw_exception_text(): + """Screening errors keep their public wording; anything else is a stable message.""" + namespace, screening_error, logged = _load_route_helper() + app = Flask(__name__) + + def held(*_args, **_kwargs): + raise screening_error("This document is unavailable pending review.", "document_held", 409) + + namespace["get_ordered_document_chunks"] = held + with app.app_context(): + response, status = namespace["_workspace_upload_file_content_response"]({}, LINKED_DOCUMENT, "user-1", "file-1") + assert status == 409 + assert response.get_json() == { + "error": "This document is unavailable pending review.", + "error_code": "document_held", + } + + def missing(*_args, **_kwargs): + raise LookupError("Document not found or access denied.") + + namespace["get_ordered_document_chunks"] = missing + with app.app_context(): + response, status = namespace["_workspace_upload_file_content_response"]({}, LINKED_DOCUMENT, "user-1", "file-1") + assert status == 404 + assert response.get_json() == {"error": "File not found in conversation"} + + def broken(*_args, **_kwargs): + raise RuntimeError("search endpoint https://secret.search.windows.net failed with key abc123") + + namespace["get_ordered_document_chunks"] = broken + with app.app_context(): + response, status = namespace["_workspace_upload_file_content_response"]({}, LINKED_DOCUMENT, "user-1", "file-1") + assert status == 500 + body = response.get_json() + assert body == {"error": "Unable to load the extracted content for this file."} + assert "secret" not in str(body) + assert logged and logged[-1][0].startswith("[GET_FILE_CONTENT]") + assert logged[-1][1]["extra"]["error_type"] == "RuntimeError" + + +def test_get_file_content_routes_workspace_uploads_before_the_empty_content_404(): + """Workspace-backed, unscreened, non-table uploads use the helper before combining text.""" + source = ROUTE_FILE.read_text(encoding="utf-8-sig") + route_start = source.index("def get_file_content():") + branch = source.index("return _workspace_upload_file_content_response(", route_start) + combine = source.index("combined_parts = []", route_start) + not_found = source.index("'File content not found'", route_start) + assert branch < combine < not_found + + guard = source[source.rindex("if (", route_start, branch):branch] + for condition in ( + "linked_document is not None", + "SCREENING_FIELD not in linked_document", + "not items[0].get('is_table')", + "not items[0].get('file_content')", + ): + assert condition in guard, f"Missing guard: {condition}" + + # Ownership is checked before any message or document read happens. + ownership = source.index("if conversation_item.get('user_id') != user_id:", route_start) + assert ownership < branch + + +def test_version_includes_the_change(): + assert_app_version_at_least("0.261.210") + + +if __name__ == "__main__": + tests = [ + test_summarize_vision_analysis_keeps_only_displayable_parts, + test_indexed_upload_returns_joined_chunks_and_details, + test_unindexed_image_falls_back_to_the_stored_vision_analysis, + test_nothing_to_show_explains_processing_or_absence, + test_route_helper_reads_chunks_in_the_linked_documents_scope, + test_route_helper_never_returns_raw_exception_text, + test_get_file_content_routes_workspace_uploads_before_the_empty_content_404, + test_version_includes_the_change, + ] + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + test() + print("Passed") + results.append(True) + except Exception as error: # noqa: BLE001 - report every failure in the runner. + print(f"Failed: {error}") + import traceback + + traceback.print_exc() + results.append(False) + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/ui_tests/test_v2_chat_image_uploads.py b/ui_tests/test_v2_chat_image_uploads.py index 329801d0d..3b835a85d 100644 --- a/ui_tests/test_v2_chat_image_uploads.py +++ b/ui_tests/test_v2_chat_image_uploads.py @@ -1,18 +1,24 @@ # test_v2_chat_image_uploads.py """ UI coverage for v2 inline chat upload image previews. -Version: 0.261.192 -Implemented in: 0.261.192 +Version: 0.261.210 +Implemented in: 0.261.192; 0.261.210 This test ensures workspace-backed uploaded image messages render as inline image cards, fall back safely when previews are unavailable, and keep non-image uploads as file chips. +Since 0.261.210 it also ensures the card is sized to the image like a generated image, its +actions are revealed on hover, Show sources and the file preview show what ingestion +extracted, and a cited image shows its extracted passage. Build: npm --prefix .\\application\\v2_ui run build -- --outDir ..\\..\\ui_tests\\artifacts\\orchestration-plan-editor Run: python -m pytest .\\ui_tests\\test_v2_chat_image_uploads.py -q """ import base64 +import json import re +import struct +import zlib from urllib.parse import parse_qs, unquote, urlsplit import pytest @@ -35,6 +41,58 @@ CONVERSATION = "image-upload-chat" +def solid_png(width, height, rgb=(96, 96, 96)): + """A real PNG of a known size, so layout can be measured against the image itself.""" + row = b"\x00" + bytes(rgb) * width + + def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + kind + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(row * height)) + chunk(b"IEND", b"") + ) + + +SIZED_PNG = solid_png(320, 200) +VISION_DESCRIPTION = "A grayscale image of a large indoor lab with a mostly empty tiled floor." +WORKSPACE_EXTRACTION = { + "file_content": ( + "=== AI Vision Analysis ===\nModel: gpt-5.4\n\n" + f"Description: {VISION_DESCRIPTION}" + ), + "filename": "lab.png", + "is_table": False, + "file_content_source": "workspace", + "indexed_text_available": True, + "workspace_document": { + "document_id": "doc-lab", + "title": "AI Vision Analysis of Indoor Environment", + "abstract": "", + "keywords": [], + "status": "Processing complete - final metadata extracted", + "percentage_complete": 100, + "number_of_pages": 1, + "extraction_engine": "document_intelligence", + "extraction_engine_reason": ( + "Content Understanding returned no content, so Document Intelligence Layout was used" + ), + "indexed_chunk_count": 1, + "vision_analysis": { + "model": "gpt-5.4", + "description": VISION_DESCRIPTION, + "objects": ["checkerboard calibration panels", "equipment rack"], + "text": "", + "analysis": "Likely a robotics or computer vision test space.", + }, + }, +} + + def file_message(message_id, filename, document_id, **overrides): return { "id": message_id, @@ -65,6 +123,10 @@ def __init__(self, assets): self.unexpected = [] self.errors = [] self.preview_hits = {} + # file_id -> (status, payload) for /api/get_file_content. + self.file_content = {} + self.file_content_requests = [] + self.cited_text = "" def handle(self, route): request = route.request @@ -78,6 +140,26 @@ def handle(self, route): route.fulfill(path=str(self.assets[path])) return self.requests.append({"path": path, "query": parsed.query, "method": request.method}) + if request.method == "POST" and path == "/api/get_file_content": + body = json.loads(request.post_data or "{}") + self.file_content_requests.append((body.get("conversation_id"), body.get("file_id"))) + status, payload = self.file_content.get( + body.get("file_id"), (404, {"error": "File content not found"}), + ) + route.fulfill(status=status, json=payload) + return + if request.method == "GET" and re.fullmatch(r"/api/message/[^/]+/metadata", path): + route.fulfill(json={"message_details": {"message_id": path.split("/")[3]}}) + return + if request.method == "GET" and path == "/api/enhanced_citations/document_metadata": + route.fulfill(json={"document_id": "doc-lab", "file_name": "lab.png", "enhanced_citations": True}) + return + if request.method == "GET" and path == "/api/enhanced_citations/image": + route.fulfill(status=200, body=SIZED_PNG, content_type="image/png") + return + if request.method == "POST" and path == "/api/get_citation": + route.fulfill(json={"cited_text": self.cited_text, "file_name": "lab.png", "page_number": 1}) + return if request.method == "POST" and path == "/upload": route.fulfill(json={ "success": True, @@ -111,6 +193,9 @@ def handle(self, route): if message_id in {"heic-image", "legacy-upload"}: route.fulfill(status=200, body=b"not-an-image", content_type="image/heic") return + if message_id == "sized-image": + route.fulfill(status=200, body=SIZED_PNG, content_type="image/png") + return route.fulfill(status=200, body=PNG, content_type="image/png") return if path == "/api/documents/doc-processing": @@ -141,7 +226,7 @@ def image_ui(editor_browser, editor_assets): assert not api.errors, api.errors -def mount_messages(page, api, messages): +def mount_messages(page, api, messages, features=None): if page.url != ORIGIN + HARNESS: page.goto(ORIGIN + HARNESS) page.wait_for_function("() => Boolean(window.OrchHarness)") @@ -156,7 +241,7 @@ def mount_messages(page, api, messages): version: '0.261.192', settings: {}, branding: { app_title: 'SimpleChat' }, - features: { enable_user_workspace: true, enable_image_generation: true }, + features: spec.features, user: { id: 'ui-user', display_name: 'Upload Tester' }, scope: { groups: [], public_workspaces: [] }, catalogs: { models: [], agents: [], prompts: [] }, @@ -175,7 +260,11 @@ def mount_messages(page, api, messages): }); H.mount('test-root', 'MessageList'); }""", - {"conversation": CONVERSATION, "messages": messages}, + { + "conversation": CONVERSATION, + "messages": messages, + "features": features or {"enable_user_workspace": True, "enable_image_generation": True}, + }, ) @@ -308,3 +397,121 @@ def test_legacy_user_upload_image_error_uses_file_style_fallback(image_ui): expect(page.get_by_text("Preview isn't available in this browser. Convert to JPG or PNG to preview it.")).to_be_visible() expect(page.locator("[id='message-legacy-upload']")).to_have_class(re.compile("items-end")) expect(page.get_by_text("legacy.heif")).to_be_visible() + + +def action_row(page): + """The hover-revealed row under the upload, found through a control it always holds.""" + return page.get_by_role("button", name="Show sources", exact=True).locator( + "xpath=ancestor::div[contains(@class, 'transition-opacity')][1]" + ) + + +def test_uploaded_image_card_is_sized_to_the_image_like_generated_images(image_ui): + page, api = image_ui + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + card = page.get_by_role("button", name="View the full-size image: lab.png") + expect(card.locator("img[alt='lab.png']")).to_be_visible() + page.wait_for_function( + "() => document.querySelector(\"img[alt='lab.png']\")?.naturalWidth === 320" + ) + box = card.bounding_box() + # Sized to the 320px image, as a generated image is, not stretched across the column. + assert box and box["width"] <= 330, box + frame = card.locator("xpath=ancestor::div[contains(@class, 'glass-flat')][1]").bounding_box() + assert frame and frame["width"] <= 345, frame + + +def test_uploaded_image_actions_are_revealed_on_hover(image_ui): + page, api = image_ui + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + card = page.get_by_role("button", name="View the full-size image: lab.png") + expect(card.locator("img[alt='lab.png']")).to_be_visible() + page.mouse.move(1, 1) + expect(action_row(page)).to_have_css("opacity", "0") + + card.hover() + expect(action_row(page)).to_have_css("opacity", "1") + for name in ("Show sources", "Message details", "Open file preview for lab.png"): + expect(page.get_by_role("button", name=name, exact=True)).to_be_visible() + + +def test_uploaded_image_sources_show_what_ingestion_extracted(image_ui): + page, api = image_ui + api.file_content["sized-image"] = (200, WORKSPACE_EXTRACTION) + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + page.get_by_role("button", name="View the full-size image: lab.png").hover() + page.get_by_role("button", name="Show sources", exact=True).click() + + expect(page.get_by_role("button", name="Sources", exact=True)).to_have_attribute("aria-pressed", "true") + vision = page.get_by_role("region", name="AI vision analysis") + expect(vision).to_be_visible() + expect(vision).to_contain_text(VISION_DESCRIPTION) + expect(vision).to_contain_text("checkerboard calibration panels") + expect(vision).to_contain_text("gpt-5.4") + expect(page.get_by_text("Azure AI Document Intelligence")).to_be_visible() + expect(page.get_by_text("Indexed text")).to_be_visible() + # The panel stays up while the pointer leaves, so it does not vanish mid-read. + page.mouse.move(1, 1) + expect(action_row(page)).to_have_css("opacity", "1") + assert api.file_content_requests == [(CONVERSATION, "sized-image")] + + page.get_by_role("button", name="Details", exact=True).click() + expect(page.get_by_role("button", name="Details", exact=True)).to_have_attribute("aria-pressed", "true") + + +def test_uploaded_image_preview_shows_extraction_instead_of_not_found(image_ui): + page, api = image_ui + api.file_content["sized-image"] = (200, WORKSPACE_EXTRACTION) + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + page.get_by_role("button", name="View the full-size image: lab.png").hover() + page.get_by_role("button", name="Open file preview for lab.png", exact=True).click() + + dialog = page.get_by_role("dialog", name="Uploaded file: lab.png") + expect(dialog).to_be_visible() + expect(dialog.get_by_role("region", name="AI vision analysis")).to_contain_text(VISION_DESCRIPTION) + expect(dialog.get_by_text("Likely a robotics or computer vision test space.")).to_be_visible() + expect(dialog.get_by_text("File content not found")).to_have_count(0) + + +def test_uploaded_image_preview_explains_a_file_that_is_still_processing(image_ui): + page, api = image_ui + api.file_content["sized-image"] = ( + 404, + {"error": "This file is still being processed. Try again when it finishes."}, + ) + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + page.get_by_role("button", name="View the full-size image: lab.png").hover() + page.get_by_role("button", name="Open file preview for lab.png", exact=True).click() + + dialog = page.get_by_role("dialog", name="Uploaded file: lab.png") + expect(dialog.get_by_text("This file is still being processed. Try again when it finishes.")).to_be_visible() + + +def test_image_citation_viewer_shows_what_was_extracted(image_ui): + page, api = image_ui + api.cited_text = f"=== AI Vision Analysis ===\nModel: gpt-5.4\n\nDescription: {VISION_DESCRIPTION}" + mount_messages( + page, + api, + [{ + "id": "answer-1", + "conversation_id": CONVERSATION, + "role": "assistant", + "content": "It is an empty lab floor. (Source: lab.png, Page: 1) [#doc-lab_1]", + "metadata": {}, + }], + features={"enable_user_workspace": True, "enable_enhanced_citations": True}, + ) + + page.get_by_role("button", name=re.compile(r"^lab\.png")).click() + + viewer = page.get_by_role("dialog", name="Cited source") + expect(viewer).to_be_visible() + expect(viewer.locator("img[alt='lab.png']")).to_be_visible() + expect(viewer.get_by_text("What was extracted from this image")).to_be_visible() + expect(viewer).to_contain_text(VISION_DESCRIPTION) From ecac24f93bdc193ce40c7ac6afceb5b1717bbaba Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 13:40:32 -0400 Subject: [PATCH 2/6] Index images without OCR text and route vision analysis through its AI connection Folded from the Development fix (#1584) for #1583: - An image whose OCR output is empty, or only a Document Intelligence page-number comment, is indexed from its AI vision description, and chunk embeddings are built from the stored text including the vision block. - Standalone images fall back to the Content Understanding image analyzer, whose Summary field is now read, before Document Intelligence Layout. - The vision model name resolves to the enabled AI connection that hosts it, for ingestion, chat image uploads and the admin vision test. React V2 adjustments: - The shared model-endpoint client helper keeps V2's order: find the model and re-check its chat capability before resolving Key Vault secrets. Metadata extraction behaves as before. - Vision analysis re-checks image support on the resolved connection model, and the legacy path keeps its existing checks. - A page that fits the embedding budget embeds its OCR text when the vision block would push it over, instead of being refused by the non-legacy embedding guard. - The V2 admin help text and the Knowledge docs describe the new behavior. (cherry picked from commit b91f9d5a153541b08c7f0ff14b9ff9381d1eabae) Refs #1583 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 4 +- application/single_app/functions_content.py | 51 +- .../functions_content_understanding.py | 61 +- application/single_app/functions_documents.py | 536 +++++--- .../functions_model_endpoint_types.py | 59 +- .../single_app/route_backend_settings.py | 10 + .../single_app/route_frontend_chats.py | 3 +- docs/admin/knowledge.md | 28 +- ...NTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md | 22 + ...SCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md | 180 +++ ...e_description_indexing_without_ocr_text.py | 1202 +++++++++++++++++ 11 files changed, 1976 insertions(+), 180 deletions(-) create mode 100644 docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md create mode 100644 functional_tests/test_image_description_indexing_without_ocr_text.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 59e1ef420..019414067 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -2320,8 +2320,8 @@ "type": "switch", "label": "Enable Multi-Modal Vision Analysis", "help": ( - "Sends page images to a vision-capable model so figures, charts and " - "scanned pages are described and indexed rather than skipped." + "Asks a vision-capable model to describe each uploaded image, so the image " + "is indexed and can be cited by what it shows even when it contains no text." ), "default": False, "role": "capability", diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index ebc0ef46d..2fa3dae30 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -625,12 +625,41 @@ def resolve_extraction_engine_for_mode(extraction_mode, settings): return functions_settings.resolve_enhanced_extraction_engine(settings) +# Persisted as the extraction engine reason and shown in workspace tooltips, so keep it short. +CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON = ( + 'Content Understanding found no text, so its image analyzer description was indexed' +) + + +def _describe_image_with_content_understanding(file_path, settings, analyze_image): + """Return the Content Understanding image analyzer description, or '' when it has none.""" + try: + description_text = str(analyze_image(file_path, settings=settings) or '').strip() + except Exception as image_analyzer_error: + log_event( + f"[EXTRACTION_ENGINE] Content Understanding image analyzer failed for " + f"{os.path.basename(file_path)}: {image_analyzer_error}. " + "Falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + return '' + + if not description_text: + log_event( + f"[EXTRACTION_ENGINE] Content Understanding image analyzer returned no description for " + f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + return description_text + + def extract_content_with_extraction_engine( file_path, extraction_mode, extraction_engine=None, settings=None, pages=None, + is_image=False, ): """Extract page content with the engine that backs the requested extraction mode. @@ -638,6 +667,11 @@ def extract_content_with_extraction_engine( prefers Azure AI Content Understanding, but always falls back to Document Intelligence ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion. + When ``is_image`` is set and the document analyzer finds no content, which is normal for a + photo with no text, the Content Understanding image analyzer's description is returned as + the image's single page instead. Only if that also yields nothing does extraction fall back + to Document Intelligence Layout. + Returns ``(pages_data, engine_used, fallback_reason)``. """ resolved_settings = settings if settings is not None else get_settings() @@ -654,7 +688,10 @@ def extract_content_with_extraction_engine( '', ) - from functions_content_understanding import extract_content_with_content_understanding + from functions_content_understanding import ( + analyze_image_with_content_understanding, + extract_content_with_content_understanding, + ) try: content_understanding_pages = extract_content_with_content_understanding( @@ -668,6 +705,18 @@ def extract_content_with_extraction_engine( functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, '', ) + if is_image: + image_description = _describe_image_with_content_understanding( + file_path, + resolved_settings, + analyze_image_with_content_understanding, + ) + if image_description: + return ( + [{'page_number': 1, 'content': image_description}], + functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON, + ) fallback_reason = ( 'Content Understanding returned no content, so Document Intelligence Layout was used' ) diff --git a/application/single_app/functions_content_understanding.py b/application/single_app/functions_content_understanding.py index c78098852..22bd8b7c2 100644 --- a/application/single_app/functions_content_understanding.py +++ b/application/single_app/functions_content_understanding.py @@ -14,6 +14,7 @@ import json import logging import os +import re import time import requests @@ -48,6 +49,11 @@ "Understanding defaults before using Enhanced extraction." ) +# Image analyzers such as prebuilt-imageSearch return a bare image reference, for example +# "![image](pages/1)", as the markdown and put the actual description in a Summary field. +MARKDOWN_IMAGE_PLACEHOLDER_PATTERN = re.compile(r'!\[[^\]]*\]\([^)]*\)') +CONTENT_UNDERSTANDING_SUMMARY_FIELD = 'summary' + class ContentUnderstandingError(Exception): """Raised when a Content Understanding request cannot be completed.""" @@ -521,8 +527,45 @@ def extract_content_with_content_understanding(file_path, pages=None, settings=N return pages_data +def _is_image_placeholder_markdown(markdown_text): + """Return True when markdown holds nothing but image references such as ``![image](pages/1)``.""" + return not MARKDOWN_IMAGE_PLACEHOLDER_PATTERN.sub('', str(markdown_text or '')).strip() + + +def _build_string_field_blocks(content): + """Return readable text for the non-empty string fields an analyzer extracted. + + The ``Summary`` field is emitted as plain text first, because it is the analyzer's own + description of the input. Every other string field is emitted as ``Name: value``. + """ + fields = content.get('fields') + if not isinstance(fields, dict): + return [] + + summary_blocks = [] + field_blocks = [] + for field_name, field_value in fields.items(): + if not isinstance(field_value, dict): + continue + value_string = field_value.get('valueString') + if not isinstance(value_string, str) or not value_string.strip(): + continue + + if str(field_name).strip().lower() == CONTENT_UNDERSTANDING_SUMMARY_FIELD: + summary_blocks.append(value_string.strip()) + else: + field_blocks.append(f"{field_name}: {value_string.strip()}") + + return summary_blocks + field_blocks + + def analyze_image_with_content_understanding(image_path, settings=None): - """Analyze a standalone image and return its description text, or an empty string.""" + """Analyze a standalone image and return its description text, or an empty string. + + Image analyzers describe the image in extracted fields rather than in the markdown, which is + only a placeholder image reference, so string fields are read alongside any real markdown + and figure descriptions. + """ resolved_settings = settings if settings is not None else _get_settings() config = _resolve_config(settings=resolved_settings) @@ -535,15 +578,27 @@ def analyze_image_with_content_understanding(image_path, settings=None): text_blocks = [] for content in _iter_document_contents(result): markdown_text = str(content.get('markdown') or '').strip() + if _is_image_placeholder_markdown(markdown_text): + markdown_text = '' if markdown_text: text_blocks.append(markdown_text) + field_blocks = _build_string_field_blocks(content) + text_blocks.extend(field_blocks) + + # De-duplicate figure descriptions against everything already emitted for this content. + emitted_text = "\n\n".join([markdown_text] + field_blocks) for summary in _build_figure_summaries(content): - rendered_summary = _render_figure_summary(summary, markdown_text) + rendered_summary = _render_figure_summary(summary, emitted_text) if rendered_summary: text_blocks.append(rendered_summary) - return "\n\n".join(block for block in text_blocks if block).strip() + description_text = "\n\n".join(block for block in text_blocks if block).strip() + debug_print( + f"[CONTENT_UNDERSTANDING] Image analyzer {config['image_analyzer_id']} returned " + f"{len(description_text)} character(s) for {os.path.basename(image_path)}" + ) + return description_text def test_content_understanding_connection(config_override, sample_file_path=None): diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index fde7909ea..27e8ce7f9 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -111,6 +111,7 @@ validate_xml_bytes, ) from functions_model_endpoint_types import ( + find_enabled_model_endpoint_for_model_name, get_model_endpoint_api_type, resolve_model_endpoint_request_model, ) @@ -309,6 +310,92 @@ def _build_model_endpoint_client( return client +def _build_model_endpoint_client_for_model( + settings, + endpoint_cfg, + model_id, + *, + fallback_endpoint_id='', + fallback_provider='', + purpose_label='model', + identity_context=None, +): + """Build a sync chat client for one model on a configured AI connection. + + Finds the model by id and re-checks that it may serve chat requests before any of the + connection's Key Vault secrets are resolved, then validates the provider and connection + details. ``purpose_label`` names the feature in error messages, which never include secrets. + + Returns ``(client, request_model)``. + """ + models = endpoint_cfg.get("models", []) or [] + model_cfg = next((m for m in models if m.get("id") == model_id), None) + if not model_cfg: + raise LookupError(f"Selected {purpose_label} model could not be found on the endpoint.") + provider = str(endpoint_cfg.get("provider") or fallback_provider or "aoai").lower() + require_model_capability(model_cfg, provider=provider) + + endpoint_cfg = keyvault_model_endpoint_get_helper( + endpoint_cfg, + endpoint_cfg.get("id") or fallback_endpoint_id, + scope="global", + return_type=SecretReturnType.VALUE, + ) + + connection = endpoint_cfg.get("connection", {}) or {} + auth_settings = endpoint_cfg.get("auth", {}) or {} + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + endpoint = str(connection.get("endpoint") or "").strip() + api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() + api_type = get_model_endpoint_api_type(endpoint_cfg) + anthropic_version = str(connection.get("anthropic_version") or "").strip() + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) + + if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: + raise ValueError(f"Selected {purpose_label} provider '{provider}' is not supported.") + if not endpoint or not deployment or ( + runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version + ): + raise ValueError(f"Selected {purpose_label} endpoint is incomplete.") + + return _build_model_endpoint_client( + auth_settings, + provider, + endpoint, + api_version, + deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get("allow_private_custom_model_endpoints", False) + ), + settings=settings, + endpoint_config=endpoint_cfg, + identity_context=identity_context, + ), deployment + + +def resolve_vision_model_endpoint(settings, vision_model): + """Return the enabled AI connection and model that serve the configured vision model. + + The vision model setting stores only a model name, so it is matched against enabled + endpoints and models when multi-endpoint models are enabled. Returns ``(endpoint, model)`` + from the normalized endpoint list, or ``(None, None)`` when the legacy GPT connection + should be used instead. + """ + settings = settings or {} + if not settings.get("enable_multi_model_endpoints", False): + return None, None + + endpoints, _ = normalize_model_endpoints(settings.get("model_endpoints", []) or []) + return find_enabled_model_endpoint_for_model_name(endpoints, vision_model) + + def _resolve_metadata_extraction_client(settings, identity_context=None): selection = _normalize_model_endpoint_selection(settings.get("metadata_extraction_model_selection")) @@ -324,56 +411,14 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): if not endpoint_cfg.get("enabled", True): raise ValueError("Selected metadata extraction endpoint is disabled.") - models = endpoint_cfg.get("models", []) or [] - model_cfg = next((m for m in models if m.get("id") == selection["model_id"]), None) - if not model_cfg: - raise LookupError("Selected metadata extraction model could not be found on the endpoint.") - provider = str(endpoint_cfg.get("provider") or "aoai").lower() - require_model_capability(model_cfg, provider=provider) - - endpoint_cfg = keyvault_model_endpoint_get_helper( + return _build_model_endpoint_client_for_model( + settings, endpoint_cfg, - endpoint_cfg.get("id") or selection["endpoint_id"], - scope="global", - return_type=SecretReturnType.VALUE, - ) - - connection = endpoint_cfg.get("connection", {}) or {} - auth_settings = endpoint_cfg.get("auth", {}) or {} - deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) - endpoint = str(connection.get("endpoint") or "").strip() - api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() - api_type = get_model_endpoint_api_type(endpoint_cfg) - anthropic_version = str(connection.get("anthropic_version") or "").strip() - runtime_protocol = infer_model_endpoint_protocol( - provider, - endpoint, - deployment, - api_type, - ) - - if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: - raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.") - if not endpoint or not deployment or ( - runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version - ): - raise ValueError("Selected metadata extraction endpoint is incomplete.") - - return _build_model_endpoint_client( - auth_settings, - provider, - endpoint, - api_version, - deployment, - api_type=api_type, - anthropic_version=anthropic_version, - allow_private_custom_endpoints=bool( - settings.get("allow_private_custom_model_endpoints", False) - ), - settings=settings, - endpoint_config=endpoint_cfg, + selection["model_id"], + fallback_endpoint_id=selection["endpoint_id"], + purpose_label="metadata extraction", identity_context=identity_context, - ), deployment + ) gpt_model = settings.get('metadata_extraction_model') if not gpt_model: @@ -428,6 +473,18 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): # treats them as a signal that Enhanced extraction is worth the extra cost, because Content # Understanding is the only engine that describes figures. DI_MARKDOWN_FIGURE_PATTERN = re.compile(r'(|!\[[^\]]*\]\()', re.IGNORECASE) +# Document Intelligence Layout emits page numbers and page breaks as markdown comments, for example +# . They annotate the page rather than carry its text, so an image whose OCR +# output is only these annotations has no text worth indexing on its own. Header and footer comments +# are deliberately not matched, because their quoted values are the recognized text itself. +DI_MARKDOWN_PAGE_ANNOTATION_PATTERN = re.compile( + r'', + re.IGNORECASE, +) +VISION_ANALYSIS_CONTENT_FIELDS = ('description', 'objects', 'text', 'analysis') +# Marks the single chunk saved for an image that has no OCR text but does have an AI vision +# description, so the save loop keeps it and save_chunks indexes the description on its own. +VISION_DESCRIPTION_ONLY_CHUNK_FLAG = 'vision_description_only' # Budget for image content merged into an existing chunk. The limit is the embedding character # budget, so a merged chunk can never grow past what the embedding endpoint accepts. OFFICE_IMAGE_MERGE_MIN_CHAR_LIMIT = 4000 @@ -816,6 +873,7 @@ def _extract_pages_with_extraction_engine( extraction_engine, settings=None, pages=None, + is_image=False, ): """Extract page content with the resolved engine, falling back to Document Intelligence Layout.""" return extract_content_with_extraction_engine( @@ -824,6 +882,7 @@ def _extract_pages_with_extraction_engine( extraction_engine=extraction_engine, settings=settings, pages=pages, + is_image=is_image, ) @@ -3805,14 +3864,77 @@ def _require_screening_chunk_write(metadata): or not is_publication(subject_from_document(metadata), marker.get("scan_id")) ): raise DocumentHeldError("Screened content must be published through its review workflow.") +def _is_usable_vision_analysis(vision_analysis): + """Return True when a vision analysis succeeded and describes something worth indexing.""" + if not isinstance(vision_analysis, dict) or 'error' in vision_analysis: + return False + + for field_name in VISION_ANALYSIS_CONTENT_FIELDS: + field_value = vision_analysis.get(field_name) + if isinstance(field_value, str): + if field_value.strip(): + return True + elif field_value: + return True + return False + + +def _format_vision_analysis_block(vision_analysis): + """Format a vision analysis as the searchable block appended to an image's chunk text. + + Returns an empty string for a missing, failed, or empty analysis, so an error message from + the vision model is never indexed as if it described the image. + """ + if not _is_usable_vision_analysis(vision_analysis): + return '' + + vision_text_parts = ["\n\n=== AI Vision Analysis ==="] + vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") + + if vision_analysis.get('description'): + vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") + + if vision_analysis.get('objects'): + objects_list = vision_analysis['objects'] + if isinstance(objects_list, list): + vision_text_parts.append(f"\nObjects Detected: {', '.join(str(item) for item in objects_list)}") + else: + vision_text_parts.append(f"\nObjects Detected: {objects_list}") + + if vision_analysis.get('text'): + vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") + + if vision_analysis.get('analysis'): + vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") + + return "\n".join(vision_text_parts) + + +def _append_vision_block_to_chunk_text(chunk_text, vision_text): + """Append a vision block to chunk text, dropping its leading blank lines when there is no text.""" + chunk_text = chunk_text or '' + if not vision_text: + return chunk_text + if not chunk_text.strip(): + return vision_text.lstrip() + return chunk_text + vision_text + + +def _has_indexable_ocr_text(page_content): + """Return True when OCR output has text beyond Document Intelligence page-number annotations.""" + text_without_annotations = DI_MARKDOWN_PAGE_ANNOTATION_PATTERN.sub('', str(page_content or '')) + return bool(text_without_annotations.strip()) def save_chunks(page_text_content, page_number, file_name, user_id, document_id, group_id=None, public_workspace_id=None): """ Save a single chunk (one page) at a time: - - Generate embedding + - Build the chunk text, appending the document's AI vision analysis when it has one + - Generate the embedding from that same text - Build chunk metadata - Upload to Search index + + A chunk with no text to index, and no vision analysis to stand in for it, is not written. """ capture = current_extraction(document_id) if capture is not None: @@ -3876,14 +3998,51 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, raise _require_screening_chunk_write(metadata) + # Build the stored chunk text before embedding it, so an image's vector reflects its AI vision + # description rather than only whatever OCR text it happened to contain. + enhanced_chunk_text = page_text_content or '' + vision_text = _format_vision_analysis_block(metadata.get('vision_analysis')) + if vision_text: + enhanced_chunk_text = _append_vision_block_to_chunk_text(enhanced_chunk_text, vision_text) + debug_print( + f"[SAVE_CHUNKS] Appended vision analysis for document {document_id}: " + f"chunk_text length {len(enhanced_chunk_text)} (original: {len(page_text_content or '')}, " + f"vision: {len(vision_text)})" + ) + else: + debug_print(f"[SAVE_CHUNKS] No usable vision analysis found for document {document_id}") + + if not enhanced_chunk_text.strip(): + log_event( + "[SAVE_CHUNKS] Skipped a chunk with no text or vision description to index.", + extra={ + "document_id": document_id, + "page_number": page_number, + "file_name": file_name, + }, + level=logging.WARNING + ) + return None # Generate embedding try: #status = f"Generating embedding for page {page_number}" #update_document(document_id=document_id, user_id=user_id, status=status) - embedding_input = page_text_content + embedding_input = enhanced_chunk_text max_embedding_characters = get_embedding_safe_chunk_characters() + # The vision block is added to the page text. When that alone pushes a page that fits + # over the budget, embed the page text by itself, as before the block was embedded, + # rather than refusing a chunk the selected embedding model could always take. + page_text = page_text_content or '' + if ( + vision_text + and len(embedding_input) > max_embedding_characters + and page_text.strip() + and len(page_text) <= max_embedding_characters + ): + embedding_input = page_text + # Last-resort guard. Every processor bounds its own chunks, so reaching this means content # tokenized far worse than estimated. Splitting is not an option here because chunk ids are # derived from the page number, so a second chunk would overwrite the first in the search @@ -3922,40 +4081,6 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, title = metadata.get('title', '') if metadata else '' document_classification = metadata.get('document_classification', 'None') if metadata else 'None' - # Check if this document has vision analysis and append it to chunk_text - vision_analysis = metadata.get('vision_analysis') - enhanced_chunk_text = page_text_content - - if vision_analysis: - debug_print(f"[SAVE_CHUNKS] Document {document_id} has vision analysis, appending to chunk_text") - # Format vision analysis as structured text for better searchability - vision_text_parts = [] - vision_text_parts.append("\n\n=== AI Vision Analysis ===") - vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") - - if vision_analysis.get('description'): - vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") - - if vision_analysis.get('objects'): - objects_list = vision_analysis['objects'] - if isinstance(objects_list, list): - vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") - else: - vision_text_parts.append(f"\nObjects Detected: {objects_list}") - - if vision_analysis.get('text'): - vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") - - if vision_analysis.get('analysis'): - vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") - - vision_text = "\n".join(vision_text_parts) - enhanced_chunk_text = page_text_content + vision_text - - debug_print(f"[SAVE_CHUNKS] Enhanced chunk_text length: {len(enhanced_chunk_text)} (original: {len(page_text_content)}, vision: {len(vision_text)})") - else: - debug_print(f"[SAVE_CHUNKS] No vision analysis found for document {document_id}") - if is_public_workspace: chunk_document = { "id": chunk_id, @@ -4110,45 +4235,43 @@ def save_chunks_batch(chunks_data, user_id, document_id, group_id=None, public_w raise _require_screening_chunk_write(metadata) + # Build every chunk's stored text first and embed that same text, matching save_chunks(). + vision_text = _format_vision_analysis_block(metadata.get('vision_analysis')) + chunks_to_save = [] + for chunk_info in chunks_data: + enhanced_chunk_text = _append_vision_block_to_chunk_text(chunk_info['page_text_content'], vision_text) + if not enhanced_chunk_text.strip(): + log_event( + "[SAVE_CHUNKS_BATCH] Skipped a chunk with no text or vision description to index.", + extra={ + "document_id": document_id, + "page_number": chunk_info.get('page_number'), + "file_name": chunk_info.get('file_name'), + }, + level=logging.WARNING + ) + continue + chunks_to_save.append((chunk_info, enhanced_chunk_text)) + + total_token_usage = {'total_tokens': 0, 'prompt_tokens': 0, 'model_deployment_name': None} + if not chunks_to_save: + return total_token_usage # Generate all embeddings in batches - texts = [c['page_text_content'] for c in chunks_data] + texts = [enhanced_chunk_text for _, enhanced_chunk_text in chunks_to_save] try: embedding_results = generate_embeddings_batch(texts) except Exception as e: log_event(f"[SAVE_CHUNKS_BATCH] Error generating batch embeddings for document {document_id}: {e}", level=logging.ERROR) raise - # Check for vision analysis once - vision_analysis = metadata.get('vision_analysis') - vision_text = "" - if vision_analysis: - vision_text_parts = [] - vision_text_parts.append("\n\n=== AI Vision Analysis ===") - vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") - if vision_analysis.get('description'): - vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") - if vision_analysis.get('objects'): - objects_list = vision_analysis['objects'] - if isinstance(objects_list, list): - vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") - else: - vision_text_parts.append(f"\nObjects Detected: {objects_list}") - if vision_analysis.get('text'): - vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") - if vision_analysis.get('analysis'): - vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") - vision_text = "\n".join(vision_text_parts) - # Build all chunk documents chunk_documents = [] - total_token_usage = {'total_tokens': 0, 'prompt_tokens': 0, 'model_deployment_name': None} - for idx, chunk_info in enumerate(chunks_data): + for idx, (chunk_info, enhanced_chunk_text) in enumerate(chunks_to_save): embedding, token_usage = embedding_results[idx] page_number = chunk_info['page_number'] file_name = chunk_info['file_name'] - page_text_content = chunk_info['page_text_content'] if token_usage: total_token_usage['total_tokens'] += token_usage.get('total_tokens', 0) @@ -4157,7 +4280,6 @@ def save_chunks_batch(chunks_data, user_id, document_id, group_id=None, public_w total_token_usage['model_deployment_name'] = token_usage.get('model_deployment_name') chunk_id = f"{document_id}_{page_number}" - enhanced_chunk_text = page_text_content + vision_text if vision_text else page_text_content if is_public_workspace: chunk_document = { @@ -6387,61 +6509,101 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): print(f"Warning: Multi-modal vision enabled but no model selected") return None - selected_models = (settings.get('gpt_model') or {}).get('selected') or [] - vision_model_record = next( - ( - model for model in selected_models if isinstance(model, dict) - and vision_model in (model.get('deploymentName'), model.get('deployment'), model.get('id')) - ), - vision_model, - ) - require_model_capability(vision_model_record) - if not is_vision_capable_model(vision_model_record): - raise ValueError('The selected vision model must support image input and text responses.') - - # Initialize client (reuse Chat Model) - enable_gpt_apim = settings.get('enable_gpt_apim', False) - debug_print(f"[VISION_ANALYSIS] Using APIM: {enable_gpt_apim}") - - if enable_gpt_apim: - api_version = settings.get('azure_apim_gpt_api_version') - endpoint = settings.get('azure_apim_gpt_endpoint') - debug_print(f"[VISION_ANALYSIS] APIM Configuration:") - debug_print(f" Endpoint: {endpoint}") - debug_print(f" API Version: {api_version}") - - gpt_client = AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=settings.get('azure_apim_gpt_subscription_key') + # The setting stores only a model name. With multi-endpoint models enabled, route the call + # through the AI connection that hosts that model; otherwise use the legacy GPT connection. + request_model = vision_model + vision_endpoint_cfg, vision_model_cfg = resolve_vision_model_endpoint(settings, vision_model) + if vision_endpoint_cfg and vision_model_cfg: + # Chat support is re-checked when the client is built; image input is checked here + # against the connection's own record, which is what the admin actually selected. + if not is_vision_capable_model(vision_model_cfg, vision_endpoint_cfg): + raise ValueError('The selected vision model must support image input and text responses.') + gpt_client, request_model = _build_model_endpoint_client_for_model( + settings, + vision_endpoint_cfg, + vision_model_cfg.get('id'), + fallback_endpoint_id=str(vision_endpoint_cfg.get('id') or ''), + purpose_label='vision', + identity_context={'user_id': user_id}, + ) + log_event( + "[VISION_ANALYSIS] Using the AI connection that hosts the vision model.", + extra={ + "document_id": document_id, + "vision_model": vision_model, + "request_model": request_model, + "endpoint_id": vision_endpoint_cfg.get('id'), + "endpoint_name": vision_endpoint_cfg.get('name'), + "model_id": vision_model_cfg.get('id'), + "provider": vision_endpoint_cfg.get('provider'), + }, ) else: - # Use managed identity or key - auth_type = settings.get('azure_openai_gpt_authentication_type', 'key') - api_version = settings.get('azure_openai_gpt_api_version') - endpoint = settings.get('azure_openai_gpt_endpoint') - - debug_print(f"[VISION_ANALYSIS] Direct Azure OpenAI Configuration:") - debug_print(f" Endpoint: {endpoint}") - debug_print(f" API Version: {api_version}") - debug_print(f" Auth Type: {auth_type}") - - if auth_type == 'managed_identity': - token_provider = get_bearer_token_provider( - DefaultAzureCredential(), - cognitive_services_scope - ) + selected_models = (settings.get('gpt_model') or {}).get('selected') or [] + vision_model_record = next( + ( + model for model in selected_models if isinstance(model, dict) + and vision_model in (model.get('deploymentName'), model.get('deployment'), model.get('id')) + ), + vision_model, + ) + require_model_capability(vision_model_record) + if not is_vision_capable_model(vision_model_record): + raise ValueError('The selected vision model must support image input and text responses.') + + # Initialize client (reuse Chat Model) + enable_gpt_apim = settings.get('enable_gpt_apim', False) + debug_print(f"[VISION_ANALYSIS] Using APIM: {enable_gpt_apim}") + + if enable_gpt_apim: + api_version = settings.get('azure_apim_gpt_api_version') + endpoint = settings.get('azure_apim_gpt_endpoint') + debug_print(f"[VISION_ANALYSIS] APIM Configuration:") + debug_print(f" Endpoint: {endpoint}") + debug_print(f" API Version: {api_version}") + gpt_client = AzureOpenAI( api_version=api_version, azure_endpoint=endpoint, - azure_ad_token_provider=token_provider + api_key=settings.get('azure_apim_gpt_subscription_key') ) else: - gpt_client = AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=settings.get('azure_openai_gpt_key') - ) + # Use managed identity or key + auth_type = settings.get('azure_openai_gpt_authentication_type', 'key') + api_version = settings.get('azure_openai_gpt_api_version') + endpoint = settings.get('azure_openai_gpt_endpoint') + + debug_print(f"[VISION_ANALYSIS] Direct Azure OpenAI Configuration:") + debug_print(f" Endpoint: {endpoint}") + debug_print(f" API Version: {api_version}") + debug_print(f" Auth Type: {auth_type}") + + if auth_type == 'managed_identity': + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + cognitive_services_scope + ) + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider + ) + else: + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=settings.get('azure_openai_gpt_key') + ) + + log_event( + "[VISION_ANALYSIS] Using the legacy GPT connection for the vision model.", + extra={ + "document_id": document_id, + "vision_model": vision_model, + "connection": "apim" if enable_gpt_apim else "azure_openai", + "multi_endpoint_enabled": bool(settings.get('enable_multi_model_endpoints', False)), + }, + ) # Create vision prompt print(f"Analyzing image with vision model: {vision_model}") @@ -6485,7 +6647,7 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): }""" api_params = { - "model": vision_model, + "model": request_model, "messages": [ { "role": "user", @@ -6629,6 +6791,15 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): except Exception as e: print(f"Error in vision analysis for {document_id}: {str(e)}") traceback.print_exc() + log_event( + "[VISION_ANALYSIS] Vision analysis failed, so no vision description was produced.", + extra={ + "document_id": document_id, + "vision_model": settings.get('multimodal_vision_model') if isinstance(settings, dict) else None, + "error_type": type(e).__name__, + }, + level=logging.WARNING, + ) return None def upload_to_blob(temp_file_path, user_id, document_id, blob_filename, update_callback, group_id=None, public_workspace_id=None, mark_enhanced_citations=True): @@ -9787,6 +9958,9 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, update_callback(num_file_chunks=num_file_chunks, status=f"Processing {original_filename} in {num_file_chunks} file chunk(s)") total_final_chunks_processed = 0 + # Only this run's vision analysis may stand in for missing OCR text. A reprocessed document can + # still carry an older analysis on its record, which must not be indexed on its own. + vision_analysis_usable = False for idx, chunk_path in enumerate(file_paths_to_process, start=1): chunk_base_name, chunk_ext_loop = os.path.splitext(original_filename) chunk_effective_filename = original_filename @@ -9851,6 +10025,7 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, extraction_mode=document_intelligence_extraction_mode, extraction_engine=extraction_engine, settings=settings, + is_image=is_image, ) if engine_fallback_reason: extraction_engine = engine_used @@ -9908,6 +10083,7 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, 'status': "AI vision analysis completed" } update_callback(**update_fields) + vision_analysis_usable = _is_usable_vision_analysis(vision_analysis) print(f"Vision analysis saved to document metadata and will be appended to chunk_text for AI Search indexing") else: print(f"Vision analysis returned no results for: {chunk_effective_filename}") @@ -9965,10 +10141,38 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, status=f"Grouped {len(final_chunks_to_save)} chunk(s) for {chunk_effective_filename} using {target_size} page(s)/slide(s) per chunk." ) elif is_image: - if di_extracted_pages: - if 'page_number' not in di_extracted_pages[0]: di_extracted_pages[0]['page_number'] = 1 - final_chunks_to_save = di_extracted_pages - else: final_chunks_to_save = [] # No text extracted + # Keep only pages with real OCR text. Layout can annotate a photo with nothing but a + # stray page-number comment, which would otherwise become the image's only chunk. + image_text_pages = [ + page for page in di_extracted_pages + if isinstance(page, dict) and _has_indexable_ocr_text(page.get('content')) + ] + if image_text_pages: + if 'page_number' not in image_text_pages[0]: + image_text_pages[0]['page_number'] = 1 + final_chunks_to_save = image_text_pages + elif vision_analysis_usable: + # No OCR text, but the vision model described the image, so index that description + # on its own. save_chunks() appends and embeds the stored vision analysis. + final_chunks_to_save = [{ + 'page_number': 1, + 'content': '', + VISION_DESCRIPTION_ONLY_CHUNK_FLAG: True, + }] + log_event( + "[VISION_ANALYSIS] Image had no OCR text, so its AI vision description was indexed on its own.", + extra={ + "document_id": document_id, + "file_name": chunk_effective_filename, + "ocr_page_count": len(di_extracted_pages), + }, + ) + update_callback( + number_of_pages=1, + status=f"No text found in image {chunk_effective_filename}; indexing its AI vision description." + ) + else: + final_chunks_to_save = [] # No text extracted and no vision description to index # --- Embedded Office image analysis (DOCX/DOC/PPTX/PPT) --- # Neither extraction engine describes figures inside Office files, so embedded images are @@ -10045,7 +10249,9 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, chunk_index = chunk_data.get("page_number", i + 1) # Ensure page number exists chunk_content = chunk_data.get("content", "") - if not chunk_content.strip(): + # A description-only image chunk has no OCR text by design; save_chunks() + # supplies its text from the stored vision analysis. + if not chunk_content.strip() and not chunk_data.get(VISION_DESCRIPTION_ONLY_CHUNK_FLAG): print(f"Skipping empty chunk index {chunk_index} for {chunk_effective_filename}.") continue @@ -13133,4 +13339,4 @@ def propagate_tags_to_chunks(document_id, tags, user_id, group_id=None, public_w except Exception as e: print(f"Error propagating tags to chunks for document {document_id}: {e}") - raise \ No newline at end of file + raise diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py index 720ecbafb..f0e658c2b 100644 --- a/application/single_app/functions_model_endpoint_types.py +++ b/application/single_app/functions_model_endpoint_types.py @@ -8,7 +8,7 @@ """ import copy -from typing import Any, Dict +from typing import Any, Dict, Optional, Tuple from functions_model_endpoint_providers import ( DEFAULT_ANTHROPIC_VERSION, @@ -33,6 +33,7 @@ "MODEL_ENDPOINT_API_TYPE_OPENAI", "MODEL_ENDPOINT_CUSTOM_API_TYPES", "MODEL_ENDPOINT_PROVIDER_CUSTOM", + "find_enabled_model_endpoint_for_model_name", "get_model_endpoint_api_type", "normalize_model_endpoint_api_type", "resolve_model_endpoint_request_model", @@ -96,3 +97,59 @@ def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: or model_data.get("name") or "" ).strip() + + +def _get_model_name_match_rank(endpoint: Dict[str, Any], model: Dict[str, Any], model_name: str) -> Optional[int]: + """Return how strongly a model matches a stored name, lower being stronger, or None.""" + deployment_name = str(model.get("deploymentName") or "").strip() + if model_name in (deployment_name, resolve_model_endpoint_request_model(endpoint, model)): + return 0 + if str(model.get("modelName") or "").strip() == model_name: + return 1 + if str(model.get("id") or "").strip() == model_name: + return 2 + return None + + +def find_enabled_model_endpoint_for_model_name( + endpoints: Any, + model_name: Any, +) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: + """Return the enabled endpoint and enabled model that serve a stored model name. + + Some settings, such as the multi-modal vision model, store only a model name rather than + an endpoint and model id. This resolves that name against the given endpoint records + without reading settings or secrets, so callers pass already-normalized endpoints. + + A deployment name or the resolved request model wins over a friendly model name, which + wins over a model id. Within the same precedence, the first match in endpoint order wins, + so a name that is a deployment on one connection and only a model family name on another + resolves to the connection that actually deploys it. Records without an ``enabled`` flag + count as enabled, matching ``normalize_model_endpoints``. + + Returns ``(endpoint, model)``, or ``(None, None)`` when nothing matches. + """ + requested_name = str(model_name or "").strip() + if not requested_name or not isinstance(endpoints, (list, tuple)): + return None, None + + best_match = (None, None) + best_rank = None + for endpoint in endpoints: + if not isinstance(endpoint, dict) or not endpoint.get("enabled", True): + continue + models = endpoint.get("models") + if not isinstance(models, (list, tuple)): + continue + for model in models: + if not isinstance(model, dict) or not model.get("enabled", True): + continue + rank = _get_model_name_match_rank(endpoint, model, requested_name) + if rank is None or (best_rank is not None and rank >= best_rank): + continue + best_match = (endpoint, model) + best_rank = rank + if rank == 0: + return best_match + + return best_match diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index adb830f69..e2ae6aa33 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -1505,6 +1505,16 @@ def _test_multimodal_vision_connection(payload): try: multi_endpoint_selection = payload.get('multi_endpoint') if isinstance(payload.get('multi_endpoint'), dict) else None + # Ingestion resolves the saved vision model name to the AI connection that hosts it, so the + # test uses that same connection whenever the lookup succeeds, even if APIM is selected. + runtime_endpoint, runtime_model = resolve_vision_model_endpoint(get_settings(), vision_model) + if runtime_endpoint and runtime_model: + multi_endpoint_selection = { + 'endpoint_id': str(runtime_endpoint.get('id') or '').strip(), + 'model_id': str(runtime_model.get('id') or '').strip(), + 'provider': str(runtime_endpoint.get('provider') or '').strip(), + 'deployment_name': resolve_model_endpoint_request_model(runtime_endpoint, runtime_model), + } if multi_endpoint_selection: settings = get_settings() identity_context = {'user_id': get_current_user_id()} diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 5b51435d4..c827d96c1 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -1518,7 +1518,8 @@ def upload_file(): extracted_content_raw, _extraction_engine_used, _extraction_engine_fallback = extract_content_with_extraction_engine( temp_file_path, extraction_mode=extraction_mode, - settings=settings + settings=settings, + is_image=is_image_file, ) # Convert pages_data list to string diff --git a/docs/admin/knowledge.md b/docs/admin/knowledge.md index 1597645d0..a261c74b3 100644 --- a/docs/admin/knowledge.md +++ b/docs/admin/knowledge.md @@ -216,10 +216,24 @@ is off by default. ### Multi-Modal Vision Analysis {#multimodal-vision-section} -Sends page images to a vision-capable model so the text inside diagrams, screenshots and -scanned pages becomes searchable alongside the extracted text. Only vision-capable -deployments are offered for selection, because a text-only model silently returns nothing -useful here. +Multi-modal vision analysis asks a vision-capable model to describe an uploaded image, list its +notable objects, and read any visible text. For workspace uploads, the description is added to the +image's search chunk and embedded with it, so an image can be found by what it shows rather than +only by the text OCR happens to read. When OCR finds no text at all, as with a photo of an empty +room, the description becomes the image's only chunk, and metadata extraction, when enabled, runs +on it. For images attached directly to a chat, the description is added to the file content the +model receives. + +Workspace images are analyzed only when Enhanced Citations is enabled. Images attached directly to +a chat are analyzed whenever this setting is on. + +The Vision Model setting stores a model name. When multi-endpoint models are enabled, SimpleChat +sends the analysis through the enabled AI connection that hosts an enabled model with that +deployment name, model name, or id, so a model deployed on a different connection than the primary +GPT resource works. A deployment name match wins over a model name match, and the first matching +connection in the list wins over later ones. When no connection matches, or multi-endpoint models +are off, the GPT connection is used, directly or through APIM. **Test Vision Analysis** uses the +same connection as ingestion. Which deployments count as vision-capable is resolved in three steps, most authoritative first: an explicit choice recorded on the model under [AI Models](ai-models.md), then the @@ -240,8 +254,8 @@ already captures the surrounding structure. | Maximum File Size (MB) | Rejects an upload larger than this before extraction runs. Applies to workspace documents and chat attachments alike. | 150 | `max_file_size_mb` | | Enable Extract Meta Data | Runs a model pass on upload to record title, authors, subject and keywords for each document. | Off | `enable_extract_meta_data`; capability toggle | | Extraction Model | The deployment the metadata pass sends its requests to. | Empty | `metadata_extraction_model` | -| Enable Multi-Modal Vision Analysis | Sends page images to a vision model so text inside diagrams and scans becomes searchable. | Off | `enable_multimodal_vision`; capability toggle | -| Vision Model | A vision-capable deployment, such as gpt-4o or a supported GPT 5 or later model. Only vision-capable models are offered. | Empty | `multimodal_vision_model` | +| Enable Multi-Modal Vision Analysis | Asks a vision model to describe each uploaded image, so an image can be found and cited by what it shows even when it contains no text. | Off | `enable_multimodal_vision`; capability toggle | +| Vision Model | A vision-capable deployment, such as gpt-4o or a supported GPT 5 or later model. Only vision-capable models are offered. With multi-endpoint models enabled, it is called through the AI connection that hosts it; otherwise through the GPT connection. | Empty | `multimodal_vision_model` | | Require DeepResearchUser App Role | Required app role value: DeepResearchUser. Assign this role to users or groups in the Enterprise App before enabling the requirement. When enabled, only assigned users can use Deep Research. | Off | `require_member_of_deep_research_user` | | Max User URLs per Turn | Direct URLs beyond this cap are recorded as omitted in the ledger. | 100 | `deep_research_max_user_urls_per_turn` | | Max Search Queries per Turn | Includes the original current-message query. | 8 | `deep_research_max_search_queries_per_turn` | @@ -256,7 +270,7 @@ already captures the surrounding structure. | Content Understanding Key | Provides the secret credential used when the selected authentication mode requires one. | Empty | `azure_content_understanding_key` | | API Version | Default: | Not specified in defaults | `azure_content_understanding_api_version` | | Document Analyzer | Default: | Not specified in defaults | `azure_content_understanding_analyzer_id` | -| Image Analyzer | Default: | Not specified in defaults | `azure_content_understanding_image_analyzer_id` | +| Image Analyzer | Content Understanding analyzer that describes images embedded in DOCX and PPTX files, and standalone images when the document analyzer finds no text in them. Its `Summary` field is indexed as the image description. | prebuilt-imageSearch | `azure_content_understanding_image_analyzer_id` | | Analyze images embedded in DOCX and PPTX files | Exposes the capability after required services, permissions, and rollout policy are ready. | On | `enable_office_embedded_image_analysis`; capability toggle | | Minimum Image Size (pixels) | Images narrower or shorter than this are skipped as icons or spacers. | Not specified in defaults | `office_embedded_image_min_pixels` | | Maximum Images Per Document | Caps per-document cost. Duplicate images are analyzed once. | Not specified in defaults | `office_embedded_image_max_per_document` | diff --git a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md index 6b18429d3..355784d02 100644 --- a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md +++ b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md @@ -45,6 +45,8 @@ flowchart TD F -->|Azure Public and configured| G[Content Understanding prebuilt-documentSearch] F -->|Gov / custom / unconfigured| J[Document Intelligence prebuilt-layout] G -->|Request fails| J + G -->|No content for an image| P[Content Understanding image analyzer] + P -->|No description| J H --> K[Extract embedded images from OOXML] K --> L{Active engine} @@ -156,6 +158,25 @@ Layout, which is the cheaper detector, and upgrades to Enhanced when the sample Images are always treated as Enhanced in Auto mode because they are single-page inputs that benefit most from figure and spatial analysis. +## Standalone Images With No Text + +`prebuilt-documentSearch` can return no pages for a photo that contains no text. When that happens +for an uploaded image, `extract_content_with_extraction_engine(..., is_image=True)` asks the image +analyzer (`azure_content_understanding_image_analyzer_id`, `prebuilt-imageSearch` by default) to +describe it, and indexes that description as page 1 with the reason *Content Understanding found no +text, so its image analyzer description was indexed*. If the image analyzer fails or returns +nothing, extraction falls back to Document Intelligence Layout as before. Workspace uploads and +images uploaded straight into a chat both use this path. PDFs and other files are not affected. + +Image analyzers return a placeholder such as `![image](pages/1)` as the markdown and put the +description in the `Summary` field. `analyze_image_with_content_understanding()` returns the +`Summary` value, any other non-empty string field as `Name: value`, any real markdown, and figure +descriptions, and ignores placeholder-only markdown. Images embedded in Office files use the same +function, so they are indexed with the description rather than the placeholder. + +Added in 0.261.047 for [#1583](https://github.com/microsoft/simplechat/issues/1583). See the +[Image Description Indexing Without OCR Text Fix](../fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md). + ## Images Inside Office Files Content Understanding accepts DOCX, DOC, and PPTX as input, but its figure analysis only applies to @@ -304,6 +325,7 @@ Enhanced extraction is disabled. | `functional_tests/test_office_embedded_image_extraction.py` | Real DOCX and PPTX zip round-trips: extraction, size and duplicate filtering, per-document cap, slide attribution, natural ordering, graceful handling of legacy and missing files, path-traversal defense against crafted entry names, and rejection of zip bombs and entity-expansion relationship parts | | `functional_tests/test_document_intelligence_pdf_image_extraction_mode.py` | Updated for the new engine split | | `functional_tests/test_document_intelligence_auto_reprocess_contract.py` | Updated for PDF and image extraction changes | +| `functional_tests/test_image_description_indexing_without_ocr_text.py` | Image analyzer `Summary` parsing, placeholder-only markdown, and the standalone image fallback before Document Intelligence Layout | ## Known Limitations diff --git a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md new file mode 100644 index 000000000..9004d754a --- /dev/null +++ b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md @@ -0,0 +1,180 @@ +# Image Description Indexing Without OCR Text Fix (v0.261.047) + +Fixed in version: **0.261.047** + +Issue: [#1583](https://github.com/microsoft/simplechat/issues/1583) + +## Issue Description + +A user uploaded two nearly identical grayscale photos of an empty lab floor. Neither photo has +real text, and one is more overexposed. + +- The less-exposed photo was indexed and got a title. +- The overexposed photo ended with the status **Processing complete - no text found in image**, + no title, and 0 chunks in Azure AI Search, so chat could not use it. + +The AI vision model had described both photos, and the overexposed photo's vision analysis was +stored on its Cosmos DB document, but no chunk was ever written for it. + +A later upload of another image failed with `404 Not Found` from +`/openai/deployments/gpt-5.6-luna/chat/completions` on the primary Azure OpenAI resource. The +selected vision model was deployed only on a different AI connection, yet the admin +**Test Vision Analysis** button reported success. + +## Root Cause Analysis + +Four problems combined: + +1. **Only OCR text could create an image chunk.** `process_di_document()` built image chunks from + OCR pages only. Content Understanding `prebuilt-documentSearch` returned no pages for either + photo, so extraction fell back to Document Intelligence `prebuilt-layout`. Layout read a strip + of floor tape on the less-exposed photo as the page number `J` (confidence 0.18), producing the + markdown ``, and returned no content for the overexposed photo. The + bogus comment became the less-exposed photo's only chunk; the overexposed photo got none. + With `total_final_chunks_processed == 0`, final metadata extraction was skipped as well. +2. **The vector ignored the vision description.** `save_chunks()` appended the vision analysis to + the stored chunk text only after generating the embedding from the OCR text, so the logs showed + `embedding - 7 tokens` for the less-exposed photo's chunk. +3. **Content Understanding's image description was discarded.** The configured image analyzer, + `prebuilt-imageSearch`, described both photos well, but it returns the markdown + `![image](pages/1)` and puts the description in `contents[0].fields.Summary.valueString`. + `analyze_image_with_content_understanding()` ignored `fields`, so it only ever returned the + placeholder, and it was only used for images embedded in Office files. +4. **The vision model ignored its AI connection.** `analyze_image_with_vision_model()` always built + its client from the legacy `azure_openai_gpt_*` (or APIM) settings, while the admin test + resolved the model's configured connection. The setting stores only the model name + (`multimodal_vision_model`), so nothing mapped it back to the connection that hosts it. + +## Technical Details + +### Files Modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_documents.py` | Shared vision block formatting, enhanced text embedded in `save_chunks()` and `save_chunks_batch()`, description-only image chunks, `is_image` threading, vision model routing, shared model-endpoint client builder | +| `application/single_app/functions_content.py` | `extract_content_with_extraction_engine(..., is_image=False)` image analyzer fallback | +| `application/single_app/functions_content_understanding.py` | `analyze_image_with_content_understanding()` reads string fields and ignores placeholder-only markdown | +| `application/single_app/functions_model_endpoint_types.py` | Pure `find_enabled_model_endpoint_for_model_name()` lookup | +| `application/single_app/route_backend_settings.py` | Test Vision Analysis uses the same connection as ingestion | +| `application/single_app/route_frontend_chats.py` | Chat image uploads pass `is_image=is_image_file` | +| `application/single_app/config.py` | Version `0.261.047` | +| `functional_tests/test_image_description_indexing_without_ocr_text.py` | New regression coverage | +| `docs/admin/knowledge.md`, `docs/admin/workspaces.md` | How vision analysis is indexed and which connection it uses; Image Analyzer role | +| `docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md` | Standalone image analyzer fallback | +| `docs/explanation/release_notes.md` | Bug Fixes entry | + +### Code Changes Summary + +**Index image descriptions when OCR finds no text** (`functions_documents.py`) + +- `_format_vision_analysis_block(vision_analysis)` replaces the two copies of the inline + formatting. Valid analyses produce exactly the previous `=== AI Vision Analysis ===` block. It + returns `''` when the analysis is missing, not a dict, has an `error` key, or has no + description, objects, text, or analysis, so a vision failure message is never indexed. +- `_append_vision_block_to_chunk_text(chunk_text, vision_text)` appends the block, dropping its + leading blank lines when there is no OCR text. +- `save_chunks()` now builds the stored text before embedding and embeds that same text, keeping + the last-resort clamp guard. It skips, with a warning, any chunk that has nothing to index. + `save_chunks_batch()` uses the same helpers so both paths stay consistent. +- `process_di_document()` records whether this run produced a usable vision analysis + (`_is_usable_vision_analysis()`). For images it keeps only OCR pages with real text, treating a + page whose only content is Document Intelligence page-number or page-break annotations, such as + ``, as having none (`_has_indexable_ocr_text()`). Header and footer + comments, such as ``, still count as text because their quoted value + is the recognized text. When no OCR text remains and a usable analysis exists, it saves one + page-1 chunk flagged with `vision_description_only`, which `save_chunks()` fills with the stored + vision block. Final metadata extraction then runs as usual. Images with real OCR text are + unchanged. Vision analysis still runs only when Enhanced Citations is enabled. + +**Fall back to the Content Understanding image analyzer** (`functions_content_understanding.py`, +`functions_content.py`) + +- `analyze_image_with_content_understanding()` emits the `Summary` field as plain text and any + other non-empty string field as `Name: value`, ignores markdown made up only of image + placeholders such as `![image](pages/1)`, and keeps the existing figure-summary handling. This + also fixes images embedded in DOCX and PPTX files, which were indexed with just the placeholder + when Content Understanding analyzed them. +- `extract_content_with_extraction_engine()` takes `is_image`. When the document analyzer returns + no content for an image, it calls the image analyzer and returns its description as page 1 with + the reason *Content Understanding found no text, so its image analyzer description was indexed*. + If the image analyzer fails or returns nothing, it logs a warning and falls back to Document + Intelligence Layout exactly as before. PDFs and other files are unchanged. Workspace uploads + pass the flag through `_extract_pages_with_extraction_engine()`, and chat uploads pass it from + `route_frontend_chats.py`. + +**Route the vision model through its AI connection** (`functions_model_endpoint_types.py`, +`functions_documents.py`, `route_backend_settings.py`) + +- `find_enabled_model_endpoint_for_model_name(endpoints, model_name)` is a pure lookup over + enabled endpoints and enabled models. A deployment name or resolved request model wins over a + model name, which wins over a model id, and the first match in endpoint order wins within the + same precedence. It returns `(endpoint, model)` or `(None, None)`. +- `resolve_vision_model_endpoint(settings, vision_model)` applies that lookup to the normalized + `model_endpoints` when multi-endpoint models are enabled. +- `_build_model_endpoint_client_for_model(settings, endpoint_cfg, model_id, ...)` holds the + client construction that metadata extraction used: Key Vault secret resolution, model lookup, + request-model resolution, API version and protocol checks, the provider allowlist, and + `_build_model_endpoint_client()`. Metadata extraction calls it with the same order and error + messages as before. +- `analyze_image_with_vision_model()` uses the matched connection and sends + `model=`. Otherwise it builds the legacy APIM or direct client exactly as + before. It logs which connection was used by name and id only, and still returns `None` on + errors. Chat image uploads call the same function. +- `_test_multimodal_vision_connection()` resolves the model name the same way, so the admin test + exercises the connection ingestion will use, including when APIM is selected for GPT. + +## Testing + +`functional_tests/test_image_description_indexing_without_ocr_text.py` runs the pure lookup tests +in-process and runs the real modules in fresh normal and optimized processes with external I/O +blocked. It covers: + +- vision block formatting for valid, error, parse-failed, and empty analyses, +- `save_chunks()` storing and embedding the same text, the description-only chunk, skipped empty + chunks, and the unchanged clamp guard, plus `save_chunks_batch()` parity, +- the less-exposed (``), overexposed (empty page), and no-page image + cases, real OCR text, header-only OCR text, mixed pages, and failed, empty, missing, or disabled + vision analysis, +- the `prebuilt-imageSearch` payload, other string fields, placeholder-only markdown, and figure + de-duplication, +- the extraction fallback for images, analyzer failure and empty results, PDFs, document analyzer + content and failure, and Standard extraction, plus the chat upload call site, +- the vision connection lookup: match precedence, disabled endpoints and models, blank and + malformed input, legacy fallback with multi-endpoint off or no match, and APIM, +- unchanged metadata extraction client construction and every error message, +- the admin vision test using the ingestion connection with an APIM payload. + +Run it with: + +```powershell +python .\functional_tests\test_image_description_indexing_without_ocr_text.py +``` + +Related suites re-run: `test_content_understanding_extraction_engine.py`, +`test_office_embedded_image_extraction.py`, `test_figure_chunk_association.py`, +`test_multimodal_vision_multi_endpoint_connection.py`, +`test_document_intelligence_pdf_image_extraction_mode.py`, +`test_document_intelligence_auto_reprocess_contract.py`, +`test_privacy_logging_telemetry_audit.py`, the model endpoint provider suites, and the route policy +suites. + +## Impact + +- Text-free images are searchable through their AI vision description and get a title from + final metadata extraction. +- Image vectors now reflect the vision description, improving retrieval for image content. +- Existing documents are not re-indexed automatically. Re-upload an affected image to index it + with this fix. **Change Extraction** also re-indexes an image, but it does not re-run final + metadata extraction, so the image still has no generated title. +- A vision model name that exists on more than one enabled connection resolves by the precedence + above, because the setting stores only the name. + +## Validation + +| Scenario | Before | After | +| --- | --- | --- | +| Overexposed photo, no OCR text | 0 chunks, no title, status "no text found in image" | 1 chunk holding the vision description, metadata extracted | +| Less-exposed photo, Layout reads `J` | Chunk text `` plus vision block, 7-token embedding | 1 chunk holding the vision description, embedded in full | +| Content Understanding image analyzer | `![image](pages/1)` | The `Summary` description | +| Vision model on another AI connection | 404 from the primary Azure OpenAI resource | Called on the connection that hosts it | +| Test Vision Analysis with APIM selected | Tested APIM while ingestion differed | Tests the connection ingestion uses | diff --git a/functional_tests/test_image_description_indexing_without_ocr_text.py b/functional_tests/test_image_description_indexing_without_ocr_text.py new file mode 100644 index 000000000..d272a2481 --- /dev/null +++ b/functional_tests/test_image_description_indexing_without_ocr_text.py @@ -0,0 +1,1202 @@ +#!/usr/bin/env python3 +# test_image_description_indexing_without_ocr_text.py +""" +Functional test for indexing image descriptions when OCR finds no text. +Version: 0.261.210 +Implemented in: 0.261.047 (Development); 0.261.210 (React V2) + +Issue #1583: two nearly identical photos of an empty lab floor had no real text. Document +Intelligence Layout returned only a bogus page-number comment for one and nothing for the other, +so the second image was never indexed and its AI vision description was unreachable from chat. +Content Understanding's image analyzer described both photos, but its description was ignored, +and the vision model was always called on the legacy GPT connection even when it was hosted on a +different AI connection. + +This test ensures that: + - the vision block keeps its existing format and is empty for failed or empty analyses, + - save_chunks and save_chunks_batch embed the same text they store, including the vision block, + and never write a chunk with nothing to index, + - an image whose OCR output is empty, or only Document Intelligence page-number annotations, is + indexed from its vision description, while images with real OCR text, including header and + footer text, are unchanged, + - the Content Understanding image analyzer's Summary field is read, and extraction falls back + to it for standalone images before Document Intelligence Layout, both for workspace uploads + and for images uploaded straight into a chat, + - the stored vision model name resolves to the enabled AI connection that hosts it, falling + back to the legacy GPT connection otherwise, and the admin test uses the same connection, + - metadata extraction builds its client exactly as before, with the same error messages. + +The model-name lookup and the chat upload call-site check run in-process. Checks that need the +real application modules run in fresh normal and optimized processes with external I/O blocked, +stubbing only the Azure-facing calls. +""" + +import ast +import copy +from contextlib import ExitStack +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +TEST_ROOT = ROOT / "functional_tests" +IMPLEMENTED_IN_VERSION = "0.261.047" + +for _path in (str(TEST_ROOT), str(APP_ROOT)): + if _path not in sys.path: + sys.path.insert(0, _path) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +# A 1x1 PNG, enough for code that only reads and base64-encodes the file. +TEST_PNG_BYTES = bytes.fromhex( + "89504e470d0a1a0a0000000d4948445200000001000000010806000000" + "1f15c4890000000d49444154789c63f8cfc0f01f0005000201e1c5a8e1" + "0000000049454e44ae426082" +) + +# Layout output for the less-exposed photo: a strip of floor tape read as the page number "J". +LESS_EXPOSED_DI_PAGES = [{"page_number": 1, "content": ''}] +# Layout output for the overexposed photo: one page with no content. +OVEREXPOSED_DI_PAGES = [{"page_number": 1, "content": ""}] + +CU_IMAGE_SUMMARY = ( + "The image shows a mostly empty indoor space with a tiled floor. In the background, there are " + "several checkerboard calibration patterns placed against the wall." +) +CU_IMAGE_SEARCH_RESULT = { + "analyzerId": "prebuilt-imageSearch", + "apiVersion": "2025-11-01", + "contents": [ + { + "kind": "document", + "markdown": "![image](pages/1)\n", + "fields": { + "Summary": {"type": "string", "valueString": CU_IMAGE_SUMMARY}, + }, + } + ], +} + +VISION_ANALYSIS = { + "description": "A grayscale photo of an empty laboratory floor with checkerboard calibration targets on the far wall.", + "objects": ["tiled floor", "checkerboard calibration target", "wall"], + "text": "", + "analysis": "The room looks like a camera calibration area in a robotics lab.", + "model": "gpt-5.4", +} +VISION_RESPONSE_JSON = { + "description": VISION_ANALYSIS["description"], + "objects": VISION_ANALYSIS["objects"], + "text": "", + "analysis": VISION_ANALYSIS["analysis"], +} + +CU_SETTINGS = { + "azure_content_understanding_endpoint": "https://content-understanding.invalid", + "azure_content_understanding_key": "content-understanding-test-key", + "azure_content_understanding_authentication_type": "key", +} + +PRIMARY_ENDPOINT_SECRET = "primary-endpoint-secret-must-not-be-logged" +GLOBAL_ENDPOINT_SECRET = "global-endpoint-secret-must-not-be-logged" + + +def _require(condition, message): + """Raise explicitly so checks survive python -O.""" + if not condition: + raise AssertionError(message) + + +def build_model_endpoints(): + """Two AI connections: the primary one, and 'Global', which alone hosts gpt-5.6-luna.""" + return [ + { + "id": "primary-endpoint", + "name": "Primary", + "provider": "aoai", + "enabled": True, + "connection": { + "endpoint": "https://simplechat-openai.invalid", + "openai_api_version": "2024-12-01-preview", + }, + "auth": {"type": "api_key", "api_key": PRIMARY_ENDPOINT_SECRET}, + "models": [ + {"id": "gpt-4o-model", "deploymentName": "gpt-4o", "modelName": "gpt-4o", "enabled": True}, + ], + }, + { + "id": "global-endpoint", + "name": "Global", + "provider": "aoai", + "enabled": True, + "connection": { + "endpoint": "https://aoai-global-team.invalid", + "openai_api_version": "2025-04-01-preview", + }, + "auth": {"type": "api_key", "api_key": GLOBAL_ENDPOINT_SECRET}, + "models": [ + { + "id": "luna-model", + "deploymentName": "luna-prod", + "modelName": "gpt-5.6-luna", + "displayName": "GPT-5.6 Luna", + "enabled": True, + }, + ], + }, + ] + + +def legacy_vision_block(vision_analysis): + """The inline formatting save_chunks used before this fix, kept here for format parity.""" + vision_text_parts = [] + vision_text_parts.append("\n\n=== AI Vision Analysis ===") + vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") + if vision_analysis.get('description'): + vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") + if vision_analysis.get('objects'): + objects_list = vision_analysis['objects'] + if isinstance(objects_list, list): + vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") + else: + vision_text_parts.append(f"\nObjects Detected: {objects_list}") + if vision_analysis.get('text'): + vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") + if vision_analysis.get('analysis'): + vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") + return "\n".join(vision_text_parts) + + +class FakeChatClient: + """Records chat completion requests and answers with a canned vision analysis.""" + + def __init__(self, content): + self.requests = [] + self._content = content + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) + + def _create(self, **params): + self.requests.append(params) + return SimpleNamespace( + id="chatcmpl-test", + model=params.get("model"), + usage=SimpleNamespace(prompt_tokens=10, completion_tokens=20, total_tokens=30), + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace(content=self._content, refusal=None), + ) + ], + ) + + +def make_recording_azure_openai(records): + """Return an AzureOpenAI stand-in that records its constructor arguments.""" + + class RecordingAzureOpenAI(FakeChatClient): + def __init__(self, **kwargs): + super().__init__(json.dumps(VISION_RESPONSE_JSON)) + self.init_kwargs = kwargs + records.append(self) + + return RecordingAzureOpenAI + + +def make_recording_client_builder(records, client=None): + """Return a build_model_endpoint_sync_chat_client stand-in that records its arguments.""" + + def fake_build(*args, **kwargs): + built_client = client or FakeChatClient(json.dumps(VISION_RESPONSE_JSON)) + records.append({"args": args, "kwargs": kwargs, "client": built_client}) + return built_client, "azure_openai" + + return fake_build + + +def write_test_image(directory, file_name="lab-floor.png"): + image_path = os.path.join(directory, file_name) + with open(image_path, "wb") as image_file: + image_file.write(TEST_PNG_BYTES) + return image_path + + +# --------------------------------------------------------------------------- +# In-process checks of the pure model-name lookup +# --------------------------------------------------------------------------- + +def _load_model_endpoint_types(): + import functions_model_endpoint_types # noqa: PLC0415 - loaded after sys.path is prepared + + return functions_model_endpoint_types + + +def test_model_name_lookup_matches_deployment_model_name_and_id(): + """A stored name matches a deployment, the request model, a model name, or a model id.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = build_model_endpoints() + + for stored_name in ("luna-prod", "gpt-5.6-luna", "luna-model", " gpt-5.6-luna "): + endpoint, model = find(endpoints, stored_name) + assert endpoint is endpoints[1], f"{stored_name!r} should resolve to the Global connection" + assert model is endpoints[1]["models"][0], f"{stored_name!r} should resolve to the Luna model" + + endpoint, model = find(endpoints, "gpt-4o") + assert endpoint is endpoints[0] and model["id"] == "gpt-4o-model" + + # OpenAI-style Custom connections send the model name, so it is their request model. + custom_endpoints = [{ + "id": "custom-endpoint", + "provider": "custom", + "api_type": "openai", + "enabled": True, + "models": [{"id": "llama-model", "modelName": "llama-3.2-vision", "enabled": True}], + }] + endpoint, model = find(custom_endpoints, "llama-3.2-vision") + assert endpoint is custom_endpoints[0] and model["id"] == "llama-model" + + +def test_model_name_lookup_prefers_deployments_then_first_endpoint(): + """Deployment and request-model matches beat model names; ties go to the first endpoint.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = [ + { + "id": "family-endpoint", + "enabled": True, + "models": [{"id": "family-model", "deploymentName": "gpt-4o-prod", "modelName": "gpt-4o", "enabled": True}], + }, + { + "id": "deployment-endpoint", + "enabled": True, + "models": [{"id": "deployment-model", "deploymentName": "gpt-4o", "enabled": True}], + }, + { + "id": "second-deployment-endpoint", + "enabled": True, + "models": [{"id": "second-deployment-model", "deploymentName": "gpt-4o", "enabled": True}], + }, + ] + + endpoint, model = find(endpoints, "gpt-4o") + assert endpoint["id"] == "deployment-endpoint", "A deployment match must beat an earlier model-name match." + assert model["id"] == "deployment-model", "The first deployment match in endpoint order must win." + + endpoints_by_name = [ + {"id": "first", "enabled": True, "models": [{"id": "a", "deploymentName": "a-prod", "modelName": "shared", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [{"id": "b", "deploymentName": "b-prod", "modelName": "shared", "enabled": True}]}, + ] + endpoint, _ = find(endpoints_by_name, "shared") + assert endpoint["id"] == "first", "Equal model-name matches must resolve to the first endpoint." + + endpoints_by_id = [ + {"id": "first", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "x-prod", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [ + {"id": "other", "deploymentName": "other-prod", "modelName": "shared-id", "enabled": True}, + ]}, + ] + endpoint, _ = find(endpoints_by_id, "shared-id") + assert endpoint["id"] == "second", "A model-name match must beat an earlier model-id match." + + endpoints_only_id = [ + {"id": "first", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "x-prod", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "y-prod", "enabled": True}]}, + ] + endpoint, _ = find(endpoints_only_id, "shared-id") + assert endpoint["id"] == "first", "Equal model-id matches must resolve to the first endpoint." + + +def test_model_name_lookup_ignores_disabled_endpoints_and_models(): + """Disabled endpoints and disabled models never match.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + assert find(disabled_endpoint, "gpt-5.6-luna") == (None, None) + + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + assert find(disabled_model, "luna-prod") == (None, None) + + fallback = [ + {"id": "off", "enabled": False, "models": [{"id": "m1", "deploymentName": "vision", "enabled": True}]}, + {"id": "on", "enabled": True, "models": [ + {"id": "m2", "deploymentName": "vision", "enabled": False}, + {"id": "m3", "deploymentName": "vision", "enabled": True}, + ]}, + ] + endpoint, model = find(fallback, "vision") + assert endpoint["id"] == "on" and model["id"] == "m3" + + +def test_model_name_lookup_handles_blank_and_malformed_input(): + """Blank names and malformed endpoint data return (None, None) instead of raising.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = build_model_endpoints() + + for model_name in (None, "", " "): + assert find(endpoints, model_name) == (None, None) + for malformed in (None, {}, "gpt-4o", 42): + assert find(malformed, "gpt-4o") == (None, None) + assert find([None, "text", {"id": "no-models", "enabled": True}, {"models": "gpt-4o"}], "gpt-4o") == (None, None) + assert find([{"id": "e", "models": [None, "gpt-4o", {"deploymentName": "gpt-4o"}]}], "gpt-4o")[1] == {"deploymentName": "gpt-4o"} + assert find(endpoints, "not-configured") == (None, None) + + before = copy.deepcopy(endpoints) + find(endpoints, "gpt-5.6-luna") + assert endpoints == before, "The lookup must not mutate the endpoint records." + + +def test_version_is_at_least_implementation_version(): + """The app version must be at or beyond the version this fix shipped in.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + + +def _call_name(call_node): + if isinstance(call_node.func, ast.Name): + return call_node.func.id + if isinstance(call_node.func, ast.Attribute): + return call_node.func.attr + return "" + + +def test_chat_uploads_request_the_image_analyzer_fallback(): + """Images uploaded straight into a chat must get the same image-analyzer fallback.""" + tree = ast.parse((APP_ROOT / "route_frontend_chats.py").read_text(encoding="utf-8-sig")) + calls = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) and _call_name(node) == "extract_content_with_extraction_engine" + ] + assert len(calls) == 1, f"Expected one chat upload extraction call site, found {len(calls)}." + + keywords = {keyword.arg: keyword.value for keyword in calls[0].keywords} + assert isinstance(keywords.get("is_image"), ast.Name) and keywords["is_image"].id == "is_image_file", ( + "The chat upload route must pass is_image=is_image_file to extract_content_with_extraction_engine()." + ) + + enclosing_function = min( + ( + node for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.lineno <= calls[0].lineno <= node.end_lineno + ), + key=lambda node: node.end_lineno - node.lineno, + ) + assert any( + isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "is_image_file" for target in node.targets) + for node in ast.walk(enclosing_function) + ), "is_image_file must be computed in the same chat upload function." + + +# --------------------------------------------------------------------------- +# Real-module checks, run by the offline probe in a fresh process +# --------------------------------------------------------------------------- + +def check_vision_block_formatting(documents): + print("Checking vision block formatting...") + block = documents._format_vision_analysis_block(VISION_ANALYSIS) + _require(block == legacy_vision_block(VISION_ANALYSIS), "A valid analysis must keep the existing vision block format.") + _require(block.startswith("\n\n=== AI Vision Analysis ==="), "The block must still start with the section header.") + + string_objects = {**VISION_ANALYSIS, "objects": "tiled floor, wall"} + _require( + documents._format_vision_analysis_block(string_objects) == legacy_vision_block(string_objects), + "A non-list objects value must keep the existing format.", + ) + + # json.loads fallback analyses carry the raw response as a description and no error key. + parse_failed = {"description": "Raw model text", "raw_response": "Raw model text", "parse_failed": True, "model": "gpt-5.4"} + _require("Description: Raw model text" in documents._format_vision_analysis_block(parse_failed), + "An analysis whose JSON could not be parsed still has a usable description.") + + unusable = [ + None, + "not a dict", + {}, + {"model": "gpt-5.4"}, + {"description": " ", "objects": [], "text": "", "analysis": None, "model": "gpt-5.4"}, + {"description": "Model returned empty content with no refusal message", + "error": "Model returned empty content with no refusal message", "model": "gpt-5.4", "parse_failed": True}, + {"error": ""}, + ] + for analysis in unusable: + _require(documents._format_vision_analysis_block(analysis) == "", + f"Unusable analysis must format to an empty string: {analysis!r}") + _require(not documents._is_usable_vision_analysis(analysis), f"Analysis must be unusable: {analysis!r}") + _require(documents._is_usable_vision_analysis({"objects": ["floor"]}), "Objects alone make an analysis usable.") + + append = documents._append_vision_block_to_chunk_text + _require(append("EXIT 12", block) == "EXIT 12" + block, "OCR text must be followed by the unchanged block.") + _require(append("", block) == block.lstrip(), "Without OCR text the block must not keep leading blank lines.") + _require(append(" \n", block).startswith("=== AI Vision Analysis ==="), "Whitespace-only text counts as no text.") + _require(append(None, "") == "" and append("EXIT 12", "") == "EXIT 12", "No block leaves the text unchanged.") + + +def check_comment_only_ocr_is_not_text(documents): + print("Checking Document Intelligence page-number-only OCR detection...") + has_text = documents._has_indexable_ocr_text + for content in ( + '', + '\n\n\n', + "", + "", + None, + " \n ", + ): + _require(not has_text(content), f"Content must count as no text: {content!r}") + for content in ( + '\nEXIT 12', + # Header and footer comments carry recognized text, so they must keep the image indexed. + '', + '\n', + "Selection marks detected:\n- Door closed: checked", + "J", + ): + _require(has_text(content), f"Content must count as real text: {content!r}") + + +def _run_save_chunks(documents, page_text, vision_analysis, max_characters=100000, legacy_embedding=True): + embedded_inputs = [] + uploaded_documents = [] + metadata = {"id": "doc-1583", "version": 1, "title": "", "tags": [], "shared_user_ids": []} + if vision_analysis is not None: + metadata["vision_analysis"] = vision_analysis + + def fake_generate_embedding(text): + embedded_inputs.append(text) + return [0.25, 0.5], {"total_tokens": 42, "prompt_tokens": 42, "model_deployment_name": "text-embedding-3-small"} + + def fake_search_write(search_client, operation_name, *args, **kwargs): + uploaded_documents.extend(kwargs.get("documents") or []) + return [{"succeeded": True}] + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: copy.deepcopy(metadata))) + stack.enter_context(patch.object(documents, "generate_embedding", fake_generate_embedding)) + stack.enter_context(patch.object(documents, "get_embedding_safe_chunk_characters", lambda *a, **k: max_characters)) + # React V2 refuses an oversized chunk for a non-legacy embedding profile instead of + # clamping it; the profile is supplied here rather than read from settings. + stack.enter_context(patch.object( + documents, "active_embedding_profile", lambda *a, **k: SimpleNamespace(legacy=legacy_embedding), + )) + stack.enter_context(patch.object(documents, "_execute_document_search_write", fake_search_write)) + stack.enter_context(patch.object(documents, "add_file_task_to_file_processing_log", lambda **_: None)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + stack.enter_context(patch.dict(documents.CLIENTS, {"search_client_user": object()})) + token_usage = documents.save_chunks( + page_text_content=page_text, + page_number=1, + file_name="lab-floor.png", + user_id="user-1", + document_id="doc-1583", + ) + return token_usage, embedded_inputs, uploaded_documents + + +def check_save_chunks_embeds_the_stored_text(documents): + print("Checking save_chunks embeds and stores the vision description...") + block = legacy_vision_block(VISION_ANALYSIS) + + token_usage, embedded, uploaded = _run_save_chunks(documents, "", VISION_ANALYSIS) + _require(len(uploaded) == 1, "A description-only image chunk must be written.") + chunk_text = uploaded[0]["chunk_text"] + _require(chunk_text == block.lstrip(), "The description-only chunk must hold exactly the vision block.") + _require(chunk_text.startswith("=== AI Vision Analysis ==="), "The chunk must not start with blank lines.") + _require(VISION_ANALYSIS["description"] in chunk_text, "The chunk must contain the vision description.") + _require(embedded == [chunk_text], "The embedding must be generated from the stored chunk text.") + _require(uploaded[0]["id"] == "doc-1583_1" and uploaded[0]["page_number"] == 1, "The chunk must be page 1.") + _require(token_usage and token_usage["total_tokens"] == 42, "Token usage must still be returned.") + + _, embedded, uploaded = _run_save_chunks(documents, "EXIT 12", VISION_ANALYSIS) + _require(uploaded[0]["chunk_text"] == "EXIT 12" + block, "OCR chunks must keep text followed by the vision block.") + _require(embedded == ["EXIT 12" + block], "The vector must reflect the description, not only the OCR text.") + + _, embedded, uploaded = _run_save_chunks(documents, "Quarterly report text.", None) + _require(uploaded[0]["chunk_text"] == "Quarterly report text.", "Chunks without vision analysis are unchanged.") + _require(embedded == ["Quarterly report text."], "Embeddings without vision analysis are unchanged.") + + error_analysis = {"description": "Model refused to respond", "error": "Model refused to respond", "model": "gpt-5.4"} + for page_text, analysis in (("", error_analysis), ("", None), (" \n", {"model": "gpt-5.4"})): + token_usage, embedded, uploaded = _run_save_chunks(documents, page_text, analysis) + _require(token_usage is None and not embedded and not uploaded, + f"A chunk with nothing to index must not be embedded or written: {page_text!r}, {analysis!r}") + + _, embedded, uploaded = _run_save_chunks(documents, "", error_analysis.copy() | {"description": ""}, max_characters=100000) + _require(not uploaded, "A failed vision analysis must never be indexed.") + + long_text = "OCR " * 40 + _, embedded, uploaded = _run_save_chunks(documents, long_text, VISION_ANALYSIS, max_characters=60) + _require(len(embedded[0]) == 60 and embedded[0] == (long_text + block)[:60], + "The last-resort guard must still clamp only the embedding input.") + _require(uploaded[0]["chunk_text"] == long_text + block, "The stored chunk text must never be clamped.") + + # React V2: a non-legacy embedding profile refuses an oversized chunk. Appending the vision + # block must not turn a page that fits into a refused one, so that page embeds its OCR text. + _, embedded, uploaded = _run_save_chunks( + documents, "EXIT 12", VISION_ANALYSIS, max_characters=60, legacy_embedding=False, + ) + _require(embedded == ["EXIT 12"], "A page that fits must embed its OCR text when the vision block does not.") + _require(uploaded[0]["chunk_text"] == "EXIT 12" + block, "The stored text must still carry the vision block.") + + refused = False + try: + _run_save_chunks(documents, long_text, VISION_ANALYSIS, max_characters=60, legacy_embedding=False) + except Exception as error: # noqa: BLE001 - the V2 guard raises AIConnectionError. + refused = "too large" in str(error) + _require(refused, "An OCR page that is itself too large is still refused, as before.") + + +def check_save_chunks_batch_matches_save_chunks(documents, content): + print("Checking save_chunks_batch embeds the same text it stores...") + block = legacy_vision_block(VISION_ANALYSIS) + batched_inputs = [] + uploaded_documents = [] + + def fake_batch(texts): + batched_inputs.append(list(texts)) + return [([0.1, 0.2], {"total_tokens": 5, "prompt_tokens": 5, "model_deployment_name": "text-embedding-3-small"}) for _ in texts] + + def fake_search_write(search_client, operation_name, *args, **kwargs): + uploaded_documents.extend(kwargs.get("documents") or []) + return [{"succeeded": True}] + + def run(metadata, chunks): + batched_inputs.clear() + uploaded_documents.clear() + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: copy.deepcopy(metadata))) + stack.enter_context(patch.object(content, "generate_embeddings_batch", fake_batch)) + stack.enter_context(patch.object(documents, "_execute_document_search_write", fake_search_write)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + stack.enter_context(patch.dict(documents.CLIENTS, {"search_client_user": object()})) + return documents.save_chunks_batch(chunks, "user-1", "doc-1583") + + chunks = [ + {"page_text_content": "Page one", "page_number": 1, "file_name": "lab-floor.png"}, + {"page_text_content": "", "page_number": 2, "file_name": "lab-floor.png"}, + ] + usage = run({"version": 1, "vision_analysis": VISION_ANALYSIS}, chunks) + stored = [document["chunk_text"] for document in uploaded_documents] + _require(stored == ["Page one" + block, block.lstrip()], "Batch chunks must use the shared vision formatting.") + _require(batched_inputs == [stored], "Batch embeddings must be generated from the stored chunk text.") + _require(usage["total_tokens"] == 10, "Batch token usage must still be accumulated.") + + usage = run({"version": 1}, chunks) + stored = [document["chunk_text"] for document in uploaded_documents] + _require(stored == ["Page one"] and batched_inputs == [["Page one"]], + "Without a vision analysis, empty batch chunks are skipped and others are unchanged.") + + usage = run({"version": 1}, [{"page_text_content": " ", "page_number": 1, "file_name": "x.md"}]) + _require(not batched_inputs and not uploaded_documents and usage["total_tokens"] == 0, + "A batch with nothing to index must not call the embedding endpoint.") + + +def _run_image_ingestion(documents, image_path, di_pages, vision_result, enable_vision=True): + saved_chunks = [] + extraction_calls = [] + status_updates = [] + vision_calls = [] + settings = { + "enable_multimodal_vision": enable_vision, + "enable_extract_meta_data": False, + "enable_enhanced_extraction": True, + "document_intelligence_pdf_image_extraction_mode": "layout", + } + + def fake_extract(file_path, extraction_mode, extraction_engine, settings=None, pages=None, is_image=False): + extraction_calls.append({"extraction_mode": extraction_mode, "extraction_engine": extraction_engine, "is_image": is_image}) + return copy.deepcopy(di_pages), "document_intelligence", "Content Understanding returned no content, so Document Intelligence Layout was used" + + def fake_vision(*args, **kwargs): + vision_calls.append(args) + return copy.deepcopy(vision_result) + + def fake_save_chunks(**kwargs): + saved_chunks.append(kwargs) + return {"total_tokens": 7, "prompt_tokens": 7, "model_deployment_name": "text-embedding-3-small"} + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_settings", lambda: dict(settings))) + stack.enter_context(patch.object(documents, "get_chunk_size_config", lambda *a, **k: {})) + stack.enter_context(patch.object(documents, "is_enhanced_extraction_enabled", lambda *a, **k: True)) + stack.enter_context(patch.object(documents, "get_effective_document_intelligence_pdf_image_extraction_mode", lambda *a, **k: "layout")) + stack.enter_context(patch.object(documents, "get_document_intelligence_auto_sample_pages", lambda *a, **k: 3)) + stack.enter_context(patch.object(documents, "_resolve_extraction_engine_for_mode", lambda *a, **k: ("content_understanding", ""))) + stack.enter_context(patch.object(documents, "upload_to_blob", lambda **_: None)) + stack.enter_context(patch.object(documents, "_extract_pages_with_extraction_engine", fake_extract)) + stack.enter_context(patch.object(documents, "analyze_image_with_vision_model", fake_vision)) + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: {"number_of_pages": 1})) + stack.enter_context(patch.object(documents, "save_chunks", fake_save_chunks)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + result = documents.process_di_document( + document_id="doc-1583", + user_id="user-1", + temp_file_path=image_path, + original_filename="lab-floor.png", + file_ext=".png", + enable_enhanced_citations=True, + update_callback=lambda **kwargs: status_updates.append(kwargs), + auto_extract_metadata=False, + ) + return result, saved_chunks, extraction_calls, status_updates, vision_calls + + +def check_text_free_images_are_indexed_from_vision(documents, work_dir): + print("Checking text-free images are indexed from their vision description...") + image_path = write_test_image(work_dir) + description_only_chunk = { + "page_text_content": "", + "page_number": 1, + "file_name": "lab-floor.png", + "user_id": "user-1", + "document_id": "doc-1583", + } + + for label, di_pages in ( + ("less-exposed photo (comment-only Layout output)", LESS_EXPOSED_DI_PAGES), + ("overexposed photo (empty Layout page)", OVEREXPOSED_DI_PAGES), + ("no pages at all", []), + ): + result, saved, extraction_calls, statuses, _ = _run_image_ingestion(documents, image_path, di_pages, VISION_ANALYSIS) + _require(result[0] == 1, f"{label}: exactly one chunk must be saved, got {result[0]}.") + _require(saved == [description_only_chunk], f"{label}: expected one description-only page-1 chunk, got {saved}.") + _require(result[1] == 7 and result[2] == "text-embedding-3-small", f"{label}: token usage must be accumulated.") + _require(extraction_calls and extraction_calls[0]["is_image"] is True, f"{label}: is_image must reach extraction.") + _require(any("indexing its AI vision description" in str(update.get("status")) for update in statuses), + f"{label}: the status must say the vision description was indexed.") + + result, saved, _, _, _ = _run_image_ingestion( + documents, image_path, [{"page_number": 1, "content": "EXIT 12"}], VISION_ANALYSIS + ) + _require(result[0] == 1 and saved[0]["page_text_content"] == "EXIT 12", + "Images with real OCR text must keep their OCR chunk unchanged.") + + header_only_pages = [{"page_number": 1, "content": ''}] + for vision_result in (VISION_ANALYSIS, None): + result, saved, _, _, _ = _run_image_ingestion(documents, image_path, header_only_pages, vision_result) + _require(result[0] == 1 and saved[0]["page_text_content"] == header_only_pages[0]["content"], + "Text Layout reports as a page header is real OCR text and must stay indexed.") + + result, saved, _, _, _ = _run_image_ingestion( + documents, + image_path, + [{"page_number": 1, "content": ''}, {"page_number": 2, "content": "Second page text"}], + VISION_ANALYSIS, + ) + _require(result[0] == 1 and [(c["page_number"], c["page_text_content"]) for c in saved] == [(2, "Second page text")], + "Only OCR pages with real text may be saved.") + + failed_analysis = {"description": "Model returned empty content with no refusal message", + "error": "Model returned empty content with no refusal message", "model": "gpt-5.4", "parse_failed": True} + for label, vision_result, enable_vision in ( + ("failed vision analysis", failed_analysis, True), + ("empty vision analysis", {"description": "", "objects": [], "text": "", "analysis": "", "model": "gpt-5.4"}, True), + ("no vision analysis", None, True), + ("vision disabled", VISION_ANALYSIS, False), + ): + result, saved, _, _, vision_calls = _run_image_ingestion( + documents, image_path, LESS_EXPOSED_DI_PAGES, vision_result, enable_vision=enable_vision + ) + _require(result[0] == 0 and not saved, f"{label}: a text-free image with no usable description must save nothing.") + if not enable_vision: + _require(not vision_calls, "Vision analysis must not run when it is disabled.") + + +def check_content_understanding_image_analyzer(content_understanding, work_dir): + print("Checking the Content Understanding image analyzer parsing...") + image_path = write_test_image(work_dir, "cu-image.png") + analyzer_calls = [] + + def analyze(result): + analyzer_calls.clear() + + def fake_analyze_file(file_path, analyzer_id=None, page_range=None, settings=None, config_override=None, max_wait_seconds=None): + analyzer_calls.append(analyzer_id) + return copy.deepcopy(result) + + with patch.object(content_understanding, "analyze_file_with_content_understanding", fake_analyze_file): + return content_understanding.analyze_image_with_content_understanding(image_path, settings=dict(CU_SETTINGS)) + + text = analyze(CU_IMAGE_SEARCH_RESULT) + _require(text == CU_IMAGE_SUMMARY, f"The Summary field must be returned as plain text, got {text!r}.") + _require(analyzer_calls == ["prebuilt-imageSearch"], f"The configured image analyzer must be used: {analyzer_calls}.") + + with_fields = copy.deepcopy(CU_IMAGE_SEARCH_RESULT) + with_fields["contents"][0]["fields"].update({ + "Setting": {"type": "string", "valueString": "Robotics calibration lab"}, + "ObjectCount": {"type": "number", "valueNumber": 4}, + "Caption": {"type": "string", "valueString": " "}, + "Tags": {"type": "array", "valueArray": [{"type": "string", "valueString": "floor"}]}, + }) + text = analyze(with_fields) + _require(text == f"{CU_IMAGE_SUMMARY}\n\nSetting: Robotics calibration lab", + f"Other string fields must be emitted as 'Name: value' after the summary, got {text!r}.") + + only_placeholder = {"contents": [{"kind": "document", "markdown": "![image](pages/1)\n![image](pages/2)"}]} + _require(analyze(only_placeholder) == "", "Placeholder-only markdown must not count as a description.") + + with_markdown = copy.deepcopy(CU_IMAGE_SEARCH_RESULT) + with_markdown["contents"][0]["markdown"] = "EXIT 12\n\n![image](pages/1)" + text = analyze(with_markdown) + _require(text == f"EXIT 12\n\n![image](pages/1)\n\n{CU_IMAGE_SUMMARY}", f"Real markdown must be kept, got {text!r}.") + + with_figures = { + "contents": [{ + "kind": "document", + "markdown": "![image](pages/1)", + "fields": {"Summary": {"type": "string", "valueString": CU_IMAGE_SUMMARY}}, + "figures": [ + {"id": "fig-1", "description": "A checkerboard calibration target.", "span": {"offset": 0, "length": 5}}, + {"id": "fig-2", "description": CU_IMAGE_SUMMARY, "span": {"offset": 0, "length": 5}}, + ], + }] + } + text = analyze(with_figures) + _require("Figure (fig-1):\nA checkerboard calibration target." in text, "Figure descriptions must still be appended.") + _require(text.count(CU_IMAGE_SUMMARY) == 1, "A figure description already in the summary must not be repeated.") + + _require(analyze({"contents": []}) == "" and analyze({}) == "", "An empty result must return an empty string.") + + +def check_extraction_engine_image_fallback(content, content_understanding, work_dir): + print("Checking the extraction engine image analyzer fallback...") + image_path = write_test_image(work_dir, "engine-image.png") + di_calls = [] + image_analyzer_calls = [] + warnings = [] + di_pages = [{"page_number": 1, "content": ''}] + no_content_reason = "Content Understanding returned no content, so Document Intelligence Layout was used" + + def run(document_pages, image_result, is_image=True, extraction_mode="layout", document_error=None): + di_calls.clear() + image_analyzer_calls.clear() + warnings.clear() + + def fake_document_analyzer(file_path, pages=None, settings=None): + if document_error: + raise document_error + return copy.deepcopy(document_pages) + + def fake_image_analyzer(file_path, settings=None): + image_analyzer_calls.append(file_path) + if isinstance(image_result, Exception): + raise image_result + return image_result + + def fake_azure_di(file_path, extraction_mode="read", pages=None): + di_calls.append(extraction_mode) + return copy.deepcopy(di_pages) + + def capture_log(message, *args, **kwargs): + warnings.append(str(message)) + + with ExitStack() as stack: + stack.enter_context(patch.object(content_understanding, "extract_content_with_content_understanding", fake_document_analyzer)) + stack.enter_context(patch.object(content_understanding, "analyze_image_with_content_understanding", fake_image_analyzer)) + stack.enter_context(patch.object(content, "extract_content_with_azure_di", fake_azure_di)) + stack.enter_context(patch.object(content, "log_event", capture_log)) + return content.extract_content_with_extraction_engine( + image_path, + extraction_mode=extraction_mode, + extraction_engine="content_understanding", + settings=dict(CU_SETTINGS), + is_image=is_image, + ) + + pages, engine, reason = run([], CU_IMAGE_SUMMARY) + _require(pages == [{"page_number": 1, "content": CU_IMAGE_SUMMARY}], f"The description must become page 1, got {pages}.") + _require(engine == "content_understanding", "The image analyzer result must be attributed to Content Understanding.") + _require(reason == content.CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON and len(reason) < 120, + "The persisted reason must name the image analyzer and stay short.") + _require(not di_calls, "Document Intelligence must not run when the image analyzer described the image.") + + for label, image_result in (("analyzer failure", RuntimeError("image analyzer unavailable")), ("empty description", " ")): + pages, engine, reason = run([], image_result) + _require(pages == di_pages and engine == "document_intelligence" and reason == no_content_reason, + f"{label}: must fall back to Document Intelligence Layout exactly as before.") + _require(di_calls == ["layout"], f"{label}: the fallback must use Layout.") + _require(any("image analyzer" in warning for warning in warnings), f"{label}: a warning must be logged.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, is_image=False) + _require(not image_analyzer_calls and di_calls == ["layout"] and reason == no_content_reason, + "PDFs and other non-image files must keep their existing fallback.") + + cu_pages = [{"page_number": 1, "content": "# Floor plan\n\nRoom 101"}] + pages, engine, reason = run(cu_pages, CU_IMAGE_SUMMARY) + _require(pages == cu_pages and engine == "content_understanding" and reason == "" and not image_analyzer_calls, + "Document analyzer content must be used as-is without calling the image analyzer.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, document_error=RuntimeError("service unavailable")) + _require(not image_analyzer_calls and engine == "document_intelligence" and reason.startswith("Content Understanding failed"), + "A document analyzer failure must keep the existing failure fallback.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, extraction_mode="read") + _require(not image_analyzer_calls and di_calls == ["read"] and engine == "document_intelligence", + "Standard extraction must stay on Document Intelligence Read.") + + +def _analyze_with_vision(documents, image_path, settings): + client_builds = [] + legacy_clients = [] + key_vault_lookups = [] + logs = [] + + def fake_key_vault(endpoint_cfg, scope_value, scope="global", return_type=None): + key_vault_lookups.append({"scope_value": scope_value, "scope": scope, "return_type": return_type}) + return endpoint_cfg + + def capture_log(message, extra=None, level=None, **kwargs): + logs.append({"message": str(message), "extra": extra or {}, "level": level}) + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "build_model_endpoint_sync_chat_client", make_recording_client_builder(client_builds))) + stack.enter_context(patch.object(documents, "keyvault_model_endpoint_get_helper", fake_key_vault)) + stack.enter_context(patch.object(documents, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + stack.enter_context(patch.object(documents, "log_event", capture_log)) + analysis = documents.analyze_image_with_vision_model(image_path, "user-1", "doc-1583", settings) + return analysis, client_builds, legacy_clients, key_vault_lookups, logs + + +def _vision_settings(**overrides): + settings = { + "enable_multimodal_vision": True, + "multimodal_vision_model": "gpt-5.6-luna", + "enable_multi_model_endpoints": True, + "model_endpoints": build_model_endpoints(), + "enable_gpt_apim": False, + "azure_openai_gpt_endpoint": "https://simplechat-openai.invalid", + "azure_openai_gpt_api_version": "2024-12-01-preview", + "azure_openai_gpt_key": "legacy-gpt-key", + "azure_openai_gpt_authentication_type": "key", + } + settings.update(overrides) + return settings + + +def check_vision_model_routes_through_its_connection(documents, work_dir): + print("Checking the vision model is routed through the AI connection that hosts it...") + image_path = write_test_image(work_dir, "vision-image.png") + + endpoint_cfg, model_cfg = documents.resolve_vision_model_endpoint(_vision_settings(), "gpt-5.6-luna") + _require(endpoint_cfg is not None and model_cfg is not None + and endpoint_cfg["id"] == "global-endpoint" and model_cfg["id"] == "luna-model", + "The stored model name must resolve to the Global connection.") + _require(documents.resolve_vision_model_endpoint(_vision_settings(enable_multi_model_endpoints=False), "gpt-5.6-luna") == (None, None), + "Without multi-endpoint models the legacy connection must be used.") + _require(documents.resolve_vision_model_endpoint(_vision_settings(), "") == (None, None), "A blank name must not match.") + + for stored_name in ("gpt-5.6-luna", "luna-prod"): + analysis, builds, legacy_clients, key_vault_lookups, logs = _analyze_with_vision( + documents, image_path, _vision_settings(multimodal_vision_model=stored_name) + ) + _require(analysis and analysis.get("description") == VISION_ANALYSIS["description"], + f"{stored_name}: the routed call must return the parsed analysis.") + _require(analysis.get("model") == stored_name, f"{stored_name}: the analysis must record the configured model name.") + _require(len(builds) == 1 and not legacy_clients, f"{stored_name}: only the Global connection client may be built.") + build_args = builds[0]["args"] + _require(build_args[1:4] == ("aoai", "https://aoai-global-team.invalid", "2025-04-01-preview"), + f"{stored_name}: wrong connection details {build_args[1:4]}.") + _require(build_args[0].get("api_key") == GLOBAL_ENDPOINT_SECRET, "The connection's own credentials must be used.") + _require(builds[0]["kwargs"].get("deployment_name") == "luna-prod", "The client must target the deployment.") + _require(builds[0]["kwargs"].get("identity_context") == {"user_id": "user-1"}, "The caller identity must be passed.") + requests = builds[0]["client"].requests + _require(len(requests) == 1 and requests[0]["model"] == "luna-prod", + f"{stored_name}: the request must send the resolved request model.") + _require(key_vault_lookups and key_vault_lookups[0]["scope_value"] == "global-endpoint", + "Key Vault-backed secrets must be resolved for the matched connection.") + routing_logs = [log for log in logs if "Using the AI connection" in log["message"]] + _require(routing_logs and routing_logs[0]["extra"].get("endpoint_id") == "global-endpoint" + and routing_logs[0]["extra"].get("endpoint_name") == "Global", + "The connection used must be logged by name and id.") + _require(GLOBAL_ENDPOINT_SECRET not in json.dumps(logs, default=str), "Connection secrets must never be logged.") + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + for label, settings in ( + ("multi-endpoint models disabled", _vision_settings(enable_multi_model_endpoints=False)), + ("no matching model", _vision_settings(multimodal_vision_model="gpt-4.1-vision")), + ("matching endpoint disabled", _vision_settings(model_endpoints=disabled_endpoint)), + ("matching model disabled", _vision_settings(model_endpoints=disabled_model)), + ): + analysis, builds, legacy_clients, _, logs = _analyze_with_vision(documents, image_path, settings) + _require(not builds and len(legacy_clients) == 1, f"{label}: the legacy GPT connection must be used.") + legacy = legacy_clients[0] + _require(legacy.init_kwargs == { + "api_version": "2024-12-01-preview", + "azure_endpoint": "https://simplechat-openai.invalid", + "api_key": "legacy-gpt-key", + }, f"{label}: the legacy client must be built exactly as before: {legacy.init_kwargs}.") + _require(legacy.requests[0]["model"] == settings["multimodal_vision_model"], + f"{label}: the legacy request must send the stored model name.") + _require(analysis and analysis.get("description") == VISION_ANALYSIS["description"], f"{label}: analysis expected.") + _require(any("legacy GPT connection" in log["message"] for log in logs), f"{label}: the legacy connection must be logged.") + + analysis, builds, legacy_clients, _, _ = _analyze_with_vision( + documents, + image_path, + _vision_settings( + enable_multi_model_endpoints=False, + enable_gpt_apim=True, + azure_apim_gpt_endpoint="https://apim.invalid", + azure_apim_gpt_api_version="2024-10-21", + azure_apim_gpt_subscription_key="apim-subscription-key", + ), + ) + _require(len(legacy_clients) == 1 and legacy_clients[0].init_kwargs == { + "api_version": "2024-10-21", + "azure_endpoint": "https://apim.invalid", + "api_key": "apim-subscription-key", + }, "The legacy APIM client must be built exactly as before.") + + incomplete = build_model_endpoints() + incomplete[1]["connection"]["endpoint"] = "" + analysis, builds, legacy_clients, _, logs = _analyze_with_vision(documents, image_path, _vision_settings(model_endpoints=incomplete)) + _require(analysis is None and not builds and not legacy_clients, "Errors on the matched connection must return None.") + _require(any("Vision analysis failed" in log["message"] for log in logs), "The failure must be logged.") + + +def _resolve_metadata_client(documents, settings): + client_builds = [] + legacy_clients = [] + key_vault_lookups = [] + + def fake_key_vault(endpoint_cfg, scope_value, scope="global", return_type=None): + key_vault_lookups.append({"scope_value": scope_value, "scope": scope, "return_type": return_type}) + return endpoint_cfg + + sentinel_client = object() + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "build_model_endpoint_sync_chat_client", make_recording_client_builder(client_builds, sentinel_client))) + stack.enter_context(patch.object(documents, "keyvault_model_endpoint_get_helper", fake_key_vault)) + stack.enter_context(patch.object(documents, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + try: + result = documents._resolve_metadata_extraction_client(settings, identity_context={"user_id": "user-1"}) + error = None + except Exception as resolution_error: # noqa: BLE001 - the check compares the exact error + result = None + error = resolution_error + return result, error, client_builds, legacy_clients, key_vault_lookups, sentinel_client + + +def check_metadata_extraction_client_is_unchanged(documents): + print("Checking the metadata extraction client is built exactly as before...") + + def settings_with(selection=None, endpoints=None, **overrides): + settings = { + "enable_multi_model_endpoints": True, + "model_endpoints": endpoints if endpoints is not None else build_model_endpoints(), + "metadata_extraction_model_selection": selection or { + "endpoint_id": "global-endpoint", + "model_id": "luna-model", + "provider": "aoai", + }, + "metadata_extraction_model": "gpt-4o", + "enable_gpt_apim": False, + "azure_openai_gpt_endpoint": "https://simplechat-openai.invalid", + "azure_openai_gpt_api_version": "2024-12-01-preview", + "azure_openai_gpt_key": "legacy-gpt-key", + "azure_openai_gpt_authentication_type": "key", + } + settings.update(overrides) + return settings + + settings = settings_with() + result, error, builds, legacy_clients, key_vault_lookups, sentinel = _resolve_metadata_client(documents, settings) + _require(error is None, f"The selected metadata model must resolve, got {error!r}.") + _require(result == (sentinel, "luna-prod"), f"Expected (client, deployment), got {result!r}.") + _require(not legacy_clients and len(builds) == 1, "Only the selected connection client may be built.") + args, kwargs = builds[0]["args"], builds[0]["kwargs"] + _require(args[1:4] == ("aoai", "https://aoai-global-team.invalid", "2025-04-01-preview"), + f"Unexpected connection details: {args[1:4]}.") + _require(args[0].get("type") == "api_key" and args[0].get("api_key") == GLOBAL_ENDPOINT_SECRET, + "The endpoint auth must be passed through.") + _require(kwargs.get("deployment_name") == "luna-prod" and kwargs.get("api_type") == "" and kwargs.get("anthropic_version") == "", + f"Unexpected client arguments: {kwargs}.") + _require(kwargs.get("allow_private_custom_endpoints") is False and kwargs.get("settings") is settings, + "The private endpoint flag and settings must be passed through.") + _require(kwargs.get("endpoint_config", {}).get("id") == "global-endpoint", "The resolved endpoint config must be passed.") + _require(kwargs.get("identity_context") == {"user_id": "user-1"}, "The identity context must be passed through.") + _require(key_vault_lookups == [{"scope_value": "global-endpoint", "scope": "global", "return_type": documents.SecretReturnType.VALUE}], + f"Key Vault secrets must be resolved exactly once for the endpoint: {key_vault_lookups}.") + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + unsupported = build_model_endpoints() + unsupported[1]["provider"] = "Bogus" + incomplete = build_model_endpoints() + incomplete[1]["connection"]["endpoint"] = "" + no_api_version = build_model_endpoints() + no_api_version[1]["connection"].pop("openai_api_version") + + # React V2 looks the model up and re-checks its chat capability before resolving any Key + # Vault secret, so a missing or disabled model is refused without a secret lookup, and a + # disabled model is refused by the shared capability check. + from functions_ai_connections import AIConnectionError # noqa: PLC0415 - offline bootstrap + + for label, settings, expected_type, expected_message, expects_key_vault in ( + ("missing endpoint", settings_with({"endpoint_id": "missing", "model_id": "luna-model"}), + LookupError, "Selected metadata extraction endpoint could not be found.", False), + ("disabled endpoint", settings_with(endpoints=disabled_endpoint), + ValueError, "Selected metadata extraction endpoint is disabled.", False), + ("missing model", settings_with({"endpoint_id": "global-endpoint", "model_id": "missing"}), + LookupError, "Selected metadata extraction model could not be found on the endpoint.", False), + ("disabled model", settings_with(endpoints=disabled_model), + AIConnectionError, "The selected model is not available for chat. Choose a compatible model.", False), + ("unsupported provider", settings_with(endpoints=unsupported), + ValueError, "Selected metadata extraction provider 'bogus' is not supported.", True), + ("incomplete endpoint", settings_with(endpoints=incomplete), + ValueError, "Selected metadata extraction endpoint is incomplete.", True), + ("missing API version", settings_with(endpoints=no_api_version), + ValueError, "Selected metadata extraction endpoint is incomplete.", True), + ): + result, error, builds, _, key_vault_lookups, _ = _resolve_metadata_client(documents, settings) + _require(type(error) is expected_type and str(error) == expected_message, + f"{label}: expected {expected_type.__name__}({expected_message!r}), got {error!r}.") + _require(not builds, f"{label}: no client may be built.") + _require(bool(key_vault_lookups) == expects_key_vault, f"{label}: Key Vault lookup order changed.") + + result, error, builds, legacy_clients, _, _ = _resolve_metadata_client( + documents, settings_with(enable_multi_model_endpoints=False) + ) + _require(error is None and not builds and len(legacy_clients) == 1 and result == (legacy_clients[0], "gpt-4o"), + "Without multi-endpoint models the legacy metadata extraction client must be used.") + _require(legacy_clients[0].init_kwargs.get("azure_endpoint") == "https://simplechat-openai.invalid" + and legacy_clients[0].init_kwargs.get("api_key") == "legacy-gpt-key", + "The legacy metadata extraction client must be built from the GPT settings.") + + result, error, _, _, _, _ = _resolve_metadata_client( + documents, settings_with(enable_multi_model_endpoints=False, metadata_extraction_model="") + ) + _require(type(error) is ValueError and str(error) == "No metadata extraction model is selected.", + "A missing legacy metadata model must keep its error.") + + +def check_vision_test_uses_the_ingestion_connection(route_settings, keyvault): + print("Checking the admin vision test uses the same connection as ingestion...") + from flask import Flask # noqa: PLC0415 - imported inside the offline bootstrap + + web = Flask("image-description-vision-test") + + def run(saved_settings, payload): + client_builds = [] + legacy_clients = [] + with ExitStack() as stack: + stack.enter_context(patch.object(route_settings, "get_settings", lambda *a, **k: copy.deepcopy(saved_settings))) + stack.enter_context(patch.object(route_settings, "get_current_user_id", lambda: "admin-user")) + stack.enter_context(patch.object(route_settings, "build_model_endpoint_sync_chat_client", + make_recording_client_builder(client_builds))) + stack.enter_context(patch.object(route_settings, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + stack.enter_context(patch.object(keyvault, "keyvault_model_endpoint_get_helper", + lambda endpoint_cfg, *a, **k: endpoint_cfg)) + with web.app_context(): + response, status_code = route_settings._test_multimodal_vision_connection(copy.deepcopy(payload)) + return response.get_json(), status_code, client_builds, legacy_clients + + apim_payload = { + "test_type": "multimodal_vision", + "enable_apim": True, + "vision_model": "gpt-5.6-luna", + "apim": { + "endpoint": "https://apim.invalid", + "subscription_key": "apim-subscription-key", + "api_version": "2024-10-21", + "deployment": "gpt-5.6-luna", + }, + } + saved_settings = _vision_settings(enable_gpt_apim=True) + + body, status_code, builds, legacy_clients = run(saved_settings, apim_payload) + _require(status_code == 200, f"The vision test must succeed, got {status_code}: {body}.") + _require(len(builds) == 1 and not legacy_clients, + "With the model hosted on an AI connection, the test must not use APIM, matching ingestion.") + _require(builds[0]["args"][2] == "https://aoai-global-team.invalid" and builds[0]["kwargs"].get("deployment_name") == "luna-prod", + "The test must call the same connection and deployment that ingestion uses.") + + direct_payload = { + "test_type": "multimodal_vision", + "enable_apim": False, + "vision_model": "gpt-5.6-luna", + "multi_endpoint": { + "endpoint_id": "global-endpoint", + "model_id": "luna-model", + "provider": "aoai", + "model_name": "gpt-5.6-luna", + "deployment_name": "gpt-5.6-luna", + }, + "direct": {"endpoint": "https://simplechat-openai.invalid", "auth_type": "key", "key": "legacy-gpt-key", + "api_version": "2024-12-01-preview", "deployment": "gpt-5.6-luna"}, + } + body, status_code, builds, legacy_clients = run(_vision_settings(), direct_payload) + _require(status_code == 200 and len(builds) == 1 and not legacy_clients + and builds[0]["args"][2] == "https://aoai-global-team.invalid", + "The classic multi-endpoint payload must keep resolving the Global connection.") + + body, status_code, builds, legacy_clients = run(_vision_settings(enable_multi_model_endpoints=False, enable_gpt_apim=True), apim_payload) + _require(status_code == 200 and not builds and len(legacy_clients) == 1 + and legacy_clients[0].init_kwargs.get("azure_endpoint") == "https://apim.invalid", + "Without multi-endpoint models the APIM test path must be unchanged.") + + +def _run_offline_probe(): + # Real application imports must occur inside the external-I/O bootstrap seam. + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline, tempfile.TemporaryDirectory() as work_dir: + import functions_content as content # noqa: PLC0415 + import functions_content_understanding as content_understanding # noqa: PLC0415 + import functions_documents as documents # noqa: PLC0415 + import functions_keyvault as keyvault # noqa: PLC0415 + import route_backend_settings as route_settings # noqa: PLC0415 + _require(not offline.network_attempts, "Application imports attempted network access.") + + check_vision_block_formatting(documents) + check_comment_only_ocr_is_not_text(documents) + check_save_chunks_embeds_the_stored_text(documents) + check_save_chunks_batch_matches_save_chunks(documents, content) + check_text_free_images_are_indexed_from_vision(documents, work_dir) + check_content_understanding_image_analyzer(content_understanding, work_dir) + check_extraction_engine_image_fallback(content, content_understanding, work_dir) + check_vision_model_routes_through_its_connection(documents, work_dir) + check_metadata_extraction_client_is_unchanged(documents) + check_vision_test_uses_the_ingestion_connection(route_settings, keyvault) + _require(not offline.network_attempts, "The checks attempted network access.") + + print("All offline image description indexing checks passed.") + + +@pytest.mark.parametrize("optimized", (False, True)) +def test_image_description_indexing_real_modules(optimized): + """Run the real-module checks in a fresh process with external I/O blocked.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(APP_ROOT), str(TEST_ROOT))) + env["PYTHONIOENCODING"] = "utf-8" + command = [sys.executable] + if optimized: + command.append("-O") + command.extend((str(Path(__file__).resolve()), "--offline-probe")) + result = subprocess.run( + command, cwd=ROOT, env=env, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=300, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +if __name__ == "__main__": + if "--offline-probe" in sys.argv: + _run_offline_probe() + else: + raise SystemExit(pytest.main([__file__, "-q"])) From 366991a5d636a9bb28eaf690b62b96c3daacccea Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 13:53:09 -0400 Subject: [PATCH 3/6] Default image generation to Azure OpenAI API version 2025-04-01-preview Folded from the Development fix (#1584): the image generation API version defaults to 2025-04-01-preview, the newest dated Azure OpenAI version and the one documented for gpt-image models, and a saved 2024-12-01-preview (the previous default) or blank value is upgraded when settings load. APIM versions are never changed. React V2 adjustments: - Image connection profiles imported with the previous default are upgraded the same way, and APIM profiles are left alone. - Unmigrated legacy Images settings with no version fall back to 2025-04-01-preview. - The V2 admin field default and the AI Models and image generation docs match. (cherry picked from commit 4882390405508fb60296d305df01055e0c0cb918) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../artifacts/admin_settings.json | 2 +- .../single_app/admin_settings_fields.py | 2 +- .../single_app/functions_image_api_route.py | 4 +- application/single_app/functions_settings.py | 80 +++- .../cosmos-settings-example.json | 2 +- docs/admin/ai-models.md | 9 +- .../IMAGE_GENERATION_RESPONSES_MODELS.md | 4 +- ...MAGE_GENERATION_API_VERSION_DEFAULT_FIX.md | 107 +++++ ...st_image_generation_api_version_default.py | 433 ++++++++++++++++++ .../test_image_generation_sdk_http.py | 7 +- 10 files changed, 637 insertions(+), 13 deletions(-) create mode 100644 docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md create mode 100644 functional_tests/test_image_generation_api_version_default.py diff --git a/application/external_apps/databaseseeder/artifacts/admin_settings.json b/application/external_apps/databaseseeder/artifacts/admin_settings.json index 2bbd3c5a1..e7408708d 100644 --- a/application/external_apps/databaseseeder/artifacts/admin_settings.json +++ b/application/external_apps/databaseseeder/artifacts/admin_settings.json @@ -37,7 +37,7 @@ "enable_image_generation": false, "enable_image_gen_apim": false, "azure_openai_image_gen_endpoint": "", - "azure_openai_image_gen_api_version": "2024-05-01-preview", + "azure_openai_image_gen_api_version": "2025-04-01-preview", "azure_openai_image_gen_authentication_type": "key", "azure_openai_image_gen_subscription_id": "", "azure_openai_image_gen_resource_group": "", diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 019414067..096a6b453 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -5909,7 +5909,7 @@ "only; a chat deployment reached through the Responses image tool uses a " "version new enough for that route regardless of what is set here." ), - "default": "2024-12-01-preview", + "default": "2025-04-01-preview", "depends_on": [ {"key": "enable_image_generation", "equals": True}, {"key": "enable_image_gen_apim", "equals": False}, diff --git a/application/single_app/functions_image_api_route.py b/application/single_app/functions_image_api_route.py index 69a1be24f..28f5fecad 100644 --- a/application/single_app/functions_image_api_route.py +++ b/application/single_app/functions_image_api_route.py @@ -41,7 +41,9 @@ # New shared connections support both Images generations and edits without inheriting # a chat API version. Explicit imported image profiles always take precedence. DEFAULT_IMAGES_API_VERSION = '2025-04-01-preview' -LEGACY_DEFAULT_IMAGES_API_VERSION = '2024-12-01-preview' +# Unmigrated legacy Images settings with no version use the same default. The previous +# fallback, 2024-12-01-preview, predates gpt-image model support. +LEGACY_DEFAULT_IMAGES_API_VERSION = DEFAULT_IMAGES_API_VERSION # What the image_generation tool call answers with when no format is stated. DEFAULT_RESPONSES_IMAGE_FORMAT = 'png' diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 7a2fa5124..ea1f4fb91 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -31,6 +31,7 @@ AIConnectionError, CAPABILITY_DEFINITIONS, EMBEDDING_SELECTION_KEY, + IMAGE_GENERATION_CAPABILITY, describe_model_capabilities, embedding_settings_use_connections, normalize_model_capability_fields, @@ -1328,6 +1329,81 @@ def normalize_tabular_parity_durable_preflight_defaults(settings): return changed +# 2025-04-01-preview is the newest dated Azure OpenAI API version, and the one Microsoft documents +# for gpt-image-1, gpt-image-1.5, and gpt-image-2. +AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT = '2025-04-01-preview' +# The previous shipped default. It predates gpt-image model support. +AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT = '2024-12-01-preview' + + +def normalize_image_generation_api_version_settings(settings): + """Upgrade a stored Azure OpenAI image generation API version the admin never customized. + + deep_merge_dicts() only fills in keys that are *missing* from a persisted settings document, + so every deployment that loaded settings before the default changed still stores the old + 2024-12-01-preview default, which predates gpt-image model support. Raising the code-level + default alone would therefore never reach those deployments. + + A stored azure_openai_image_gen_api_version that is exactly the previous default, or blank, + was never chosen by an admin, so it is upgraded to AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT. + Any other value is a deliberate admin choice and is left untouched. + + azure_apim_image_gen_api_version is intentionally not migrated. An APIM gateway defines the + API version it accepts, which is why that setting has no default at all; rewriting it could + break a working gateway. + + Returns True when the settings were changed. + """ + if not isinstance(settings, dict): + return False + + stored_version = settings.get('azure_openai_image_gen_api_version') + if stored_version is not None and not isinstance(stored_version, str): + return False + + normalized_version = (stored_version or '').strip() + if normalized_version and normalized_version != AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT: + return False + + settings['azure_openai_image_gen_api_version'] = AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT + return True + + +def normalize_image_connection_api_version_settings(settings): + """Upgrade image connection profiles that still carry the previous default API version. + + Importing legacy image settings into AI Connections copied the stored Images API version + into the imported connection's image operation profile, so every connection imported + while 2024-12-01-preview was the stored default still sends it. That value was never an + admin choice, so it is upgraded to AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT. Any other + version is left as it is, and a blank one already falls back to the newer default at + request time. + + A profile behind an API Management gateway is never changed, for the same reason the APIM + legacy setting is not: the gateway defines the versions it accepts. + + Returns True when the settings were changed. + """ + if not isinstance(settings, dict) or not isinstance(settings.get('model_endpoints'), list): + return False + + changed = False + for endpoint in settings['model_endpoints']: + connection = endpoint.get('connection') if isinstance(endpoint, dict) else None + profiles = connection.get('operation_settings') if isinstance(connection, dict) else None + profile = profiles.get(IMAGE_GENERATION_CAPABILITY) if isinstance(profiles, dict) else None + if not isinstance(profile, dict) or profile.get('is_apim'): + continue + stored_version = profile.get('api_version') + if ( + isinstance(stored_version, str) + and stored_version.strip() == AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT + ): + profile['api_version'] = AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT + changed = True + return changed + + def get_settings(use_cosmos=False, include_source=False): default_settings = { # External health check @@ -1593,7 +1669,7 @@ def get_settings(use_cosmos=False, include_source=False): 'enable_image_generation': False, 'enable_image_gen_apim': False, 'azure_openai_image_gen_endpoint': '', - 'azure_openai_image_gen_api_version': '2024-12-01-preview', + 'azure_openai_image_gen_api_version': AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT, 'azure_openai_image_gen_authentication_type': 'key', 'azure_openai_image_gen_subscription_id': '', 'azure_openai_image_gen_resource_group': '', @@ -2061,6 +2137,8 @@ def normalize_loaded_settings(settings_item): normalize_key_vault_reminder_settings(merged) normalize_model_endpoint_identity_header_settings(merged) normalize_tabular_parity_durable_preflight_defaults(merged) + normalize_image_generation_api_version_settings(merged) + normalize_image_connection_api_version_settings(merged) merged['enable_tabular_processing_plugin'] = is_tabular_processing_enabled(merged) diff --git a/artifacts/cosmos_examples/cosmos-settings-example.json b/artifacts/cosmos_examples/cosmos-settings-example.json index 526e6ed71..434412928 100644 --- a/artifacts/cosmos_examples/cosmos-settings-example.json +++ b/artifacts/cosmos_examples/cosmos-settings-example.json @@ -39,7 +39,7 @@ }, "enable_image_generation": true, "azure_openai_image_gen_endpoint": "", - "azure_openai_image_gen_api_version": "2024-12-01-preview", + "azure_openai_image_gen_api_version": "2025-04-01-preview", "azure_openai_image_gen_authentication_type": "key", "azure_openai_image_gen_subscription_id": "", "azure_openai_image_gen_resource_group": "", diff --git a/docs/admin/ai-models.md b/docs/admin/ai-models.md index c8d4b1f09..840809ef8 100644 --- a/docs/admin/ai-models.md +++ b/docs/admin/ai-models.md @@ -397,8 +397,11 @@ MAI preview status is shown, and optional Bing web grounding is not enabled. New shared Images connections without an image-specific API version use `2025-04-01-preview` for generations and edits. Explicit image operation versions, -including older imported versions, are preserved. Unmigrated legacy Images settings -retain their `2024-12-01-preview` fallback when no image version is configured. +including older imported versions, are preserved, except the previous default +`2024-12-01-preview`, which predates gpt-image support: since **0.261.210** a direct +(non-APIM) image profile or legacy image setting still holding that exact value is +upgraded to `2025-04-01-preview` when settings load. Unmigrated legacy Images settings +with no version configured also use `2025-04-01-preview`. These versions do not come from the chat connection's API version or legacy root settings after shared selection. Direct OpenAI uses `/v1` without an Azure @@ -462,7 +465,7 @@ model-selection API is unchanged. | Resource Group | Direct-resource discovery context | Empty | `azure_openai_image_gen_resource_group` | | Azure OpenAI Image Generation Key | Direct-route credential, through scoped secret helpers | Empty | `azure_openai_image_gen_key` | | Image model | Active direct deployment and saved model metadata | None selected | `image_gen_model` | -| Azure OpenAI Image Gen API Version | Legacy direct Images operation version; independent of chat and of newly selected shared image defaults | 2024-12-01-preview | `azure_openai_image_gen_api_version` | +| Azure OpenAI Image Gen API Version | Legacy direct Images operation version; independent of chat and of newly selected shared image defaults. A saved `2024-12-01-preview`, the previous default, or a blank value is upgraded to `2025-04-01-preview` when settings load. | 2025-04-01-preview | `azure_openai_image_gen_api_version` | | Azure APIM Endpoint | Gateway address and path | Empty | `azure_apim_image_gen_endpoint` | | Azure APIM API Version | Gateway image operation version | Empty | `azure_apim_image_gen_api_version` | | Azure APIM Deployment | Active gateway deployment name | Empty | `azure_apim_image_gen_deployment` | diff --git a/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md b/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md index e8e2b663f..29fc4318e 100644 --- a/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md +++ b/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md @@ -79,8 +79,8 @@ Images and Responses have separate version contracts: | Configuration | Version used | | --- | --- | | Shared Images model with no image-specific version, including a missing operation profile | `2025-04-01-preview` for Images generations and edits | -| Shared Images model with an explicit `connection.operation_settings.image_generation.api_version` | The stored version, including an older imported version | -| Unmigrated legacy Images route without a configured image API version | `2024-12-01-preview` | +| Shared Images model with an explicit `connection.operation_settings.image_generation.api_version` | The stored version, including an older imported version. Since 0.261.210 a direct profile still holding the previous default `2024-12-01-preview` is upgraded to `2025-04-01-preview` when settings load. | +| Unmigrated legacy Images route without a configured image API version | `2025-04-01-preview` (`2024-12-01-preview` before 0.261.210) | | Direct OpenAI Images or Responses | OpenAI `/v1`, without an Azure version query | | MAI Image | `/mai/v1`, without an Azure OpenAI version query | | Foundry FLUX | `api-version=preview`; native per-model paths or the documented v1 Images-compatible Kontext paths | diff --git a/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md new file mode 100644 index 000000000..7cbb364f1 --- /dev/null +++ b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md @@ -0,0 +1,107 @@ +# Image Generation API Version Default Fix (v0.261.047) + +Fixed in version: **0.261.047** (Development) and **0.261.210** (React V2) + +## React V2 + +React V2 manages image generation through AI Connections, so the same change reaches three more +places in **0.261.210**: + +- `normalize_image_connection_api_version_settings()` in `functions_settings.py` runs on every + settings load. It upgrades a connection's image operation profile + (`connection.operation_settings.image_generation.api_version`) that is exactly + `2024-12-01-preview`, which the legacy image import copied from the old default. APIM + profiles, other versions and blank versions are left alone. A blank version already falls back + to `2025-04-01-preview` at request time. +- `LEGACY_DEFAULT_IMAGES_API_VERSION` in `functions_image_api_route.py`, the fallback for + unmigrated legacy Images settings with no version, now equals `DEFAULT_IMAGES_API_VERSION` + (`2025-04-01-preview`). +- The V2 admin field for `azure_openai_image_gen_api_version` in `admin_settings_fields.py` + shows the new default, and `docs/admin/ai-models.md` and + `docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md` describe the upgrade. + +`functional_tests/test_image_generation_api_version_default.py` covers the connection profile +migration, including a real settings load that persists it, and +`functional_tests/test_image_generation_sdk_http.py` expects the new fallback on the wire. + +## Issue Description + +The Azure OpenAI image generation API version, `azure_openai_image_gen_api_version`, defaulted to +`2024-12-01-preview`. That version predates gpt-image model support, so a deployment that left the +default in place could not use gpt-image-1, gpt-image-1.5, or gpt-image-2 without an admin +changing it by hand. + +## Root Cause Analysis + +The shipped default, `2024-12-01-preview`, predates gpt-image model support; Microsoft documents +`2025-04-01-preview` for the gpt-image models. Raising the code-level default alone would not have +been enough: `get_settings()` merges defaults with `deep_merge_dicts()`, which only fills keys that +are missing from the persisted settings document. Every deployment that had already loaded +settings stored `2024-12-01-preview`, and would have kept it indefinitely. + +## Technical Details + +### Files Modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_settings.py` | New default constant and load-time migration | +| `docs/admin/ai-models.md` | Image Gen API Version default | +| `artifacts/cosmos_examples/cosmos-settings-example.json` | Image generation API version example | +| `application/external_apps/databaseseeder/artifacts/admin_settings.json` | Image generation API version seed value | +| `application/single_app/config.py` | Version `0.261.047` | +| `functional_tests/test_image_generation_api_version_default.py` | New regression coverage | +| `docs/explanation/release_notes.md` | Bug Fixes entry | + +### Code Changes Summary + +- `AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT = '2025-04-01-preview'` is the new default in + `get_settings()`. It is the newest dated Azure OpenAI API version and the one Microsoft documents + for gpt-image-1, gpt-image-1.5, and gpt-image-2. +- `AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT = '2024-12-01-preview'` records the old + default for the migration. +- `normalize_image_generation_api_version_settings(settings)` runs from + `normalize_loaded_settings()` on every settings load. It upgrades a stored value that is exactly + the previous default, or blank, to the new default and returns `True` so the change is + persisted. Any other value is a deliberate admin choice and is left untouched. +- `azure_apim_image_gen_api_version` is intentionally not migrated. An APIM gateway defines the API + version it accepts, which is why that setting has no default. + +A saved `2024-12-01-preview` cannot be told apart from the untouched old default, so that exact +value is always upgraded. Any other saved version is kept, so an admin can still pin a specific +version by entering it. + +## Testing + +`functional_tests/test_image_generation_api_version_default.py` covers: + +- the new default and the named constants, +- the previous default and padded previous default being upgraded, +- blank, whitespace, missing, and `None` values being upgraded, +- custom values such as `2025-03-01-preview`, and the new default itself, being left untouched, +- the APIM API version never changing, +- non-dict settings and non-string values being handled safely, +- the migration being wired into `normalize_loaded_settings()`, +- the admin docs and example artifacts showing the new default, +- a real `get_settings()` load, in fresh normal and optimized processes with external I/O blocked, + that migrates and persists the value to the settings document. + +Run it with: + +```powershell +python .\functional_tests\test_image_generation_api_version_default.py +``` + +The existing settings suites `test_app_settings_store_consistency.py`, +`test_settings_deep_merge_persistence_fix.py`, `test_tabular_parity_stale_settings_migration.py`, +and `test_app_settings_cache_versioning.py` were re-run and pass. + +## Validation + +| Stored `azure_openai_image_gen_api_version` | Before | After | +| --- | --- | --- | +| `2024-12-01-preview` | Kept | `2025-04-01-preview` | +| Blank | Kept blank | `2025-04-01-preview` | +| `2025-03-01-preview` or any other value | Kept | Kept | +| New deployment | `2024-12-01-preview` | `2025-04-01-preview` | +| `azure_apim_image_gen_api_version` | Kept | Kept | diff --git a/functional_tests/test_image_generation_api_version_default.py b/functional_tests/test_image_generation_api_version_default.py new file mode 100644 index 000000000..9389b31d0 --- /dev/null +++ b/functional_tests/test_image_generation_api_version_default.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +# test_image_generation_api_version_default.py +""" +Functional test for the Azure OpenAI image generation API version default and its migration. +Version: 0.261.210 +Implemented in: 0.261.047 (Development); 0.261.210 (React V2) + +azure_openai_image_gen_api_version defaulted to 2024-12-01-preview, which predates gpt-image +model support. The default is now 2025-04-01-preview, the newest dated Azure OpenAI API version and +the one Microsoft documents for gpt-image-1, gpt-image-1.5, and gpt-image-2. deep_merge_dicts() +never overwrites a key that a persisted settings document already has, so existing deployments +keep the old default unless it is migrated when settings load. + +This test ensures that: + - the shipped default is 2025-04-01-preview, + - normalize_image_generation_api_version_settings() upgrades the previous default and blank + values, leaves any other value untouched, and never changes the APIM API version, + - the migration runs on every settings load and is persisted to the settings document, + - the admin documentation and example settings artifacts show the new default. + +React V2 also stores image settings on AI Connections. There this test also ensures that: + - imported image connection profiles still holding the previous default are upgraded, while + APIM profiles, other versions, and blank versions are left alone, + - unmigrated legacy Images settings with no version fall back to the new default, + - the V2 admin field shows the new default. + +The migration helper is loaded from source without importing functions_settings, which builds +Azure clients at import time. The end-to-end load check imports the real module in fresh normal +and optimized processes with external I/O blocked. +""" + +import ast +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +TEST_ROOT = ROOT / "functional_tests" +SETTINGS_FILE = APP_ROOT / "functions_settings.py" +IMPLEMENTED_IN_VERSION = "0.261.047" +NEW_DEFAULT = "2025-04-01-preview" +PREVIOUS_DEFAULT = "2024-12-01-preview" +MIGRATION_FUNCTION = "normalize_image_generation_api_version_settings" +DEFAULT_CONSTANT = "AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT" +PREVIOUS_DEFAULT_CONSTANT = "AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT" + +if str(TEST_ROOT) not in sys.path: + sys.path.insert(0, str(TEST_ROOT)) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +def _require(condition, message): + """Raise explicitly so checks survive python -O.""" + if not condition: + raise AssertionError(message) + + +def load_migration(): + """Load the migration helper and its constants from source, without importing Azure config.""" + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + wanted_constants = {DEFAULT_CONSTANT, PREVIOUS_DEFAULT_CONSTANT} + + selected_nodes = [] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == MIGRATION_FUNCTION: + selected_nodes.append(node) + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in wanted_constants + ): + selected_nodes.append(node) + + assert len(selected_nodes) == 3, ( + f"Expected {DEFAULT_CONSTANT}, {PREVIOUS_DEFAULT_CONSTANT}, and {MIGRATION_FUNCTION}() " + "at module level in functions_settings.py" + ) + + namespace = {} + exec(compile(ast.Module(body=selected_nodes, type_ignores=[]), str(SETTINGS_FILE), "exec"), namespace) + return namespace + + +def _function_node(tree, name): + return next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == name) + + +def test_default_is_the_newest_gpt_image_api_version(): + """The code-level default must be 2025-04-01-preview.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + namespace = load_migration() + assert namespace[DEFAULT_CONSTANT] == NEW_DEFAULT + assert namespace[PREVIOUS_DEFAULT_CONSTANT] == PREVIOUS_DEFAULT + + source = SETTINGS_FILE.read_text(encoding="utf-8") + assert f"'azure_openai_image_gen_api_version': {DEFAULT_CONSTANT}," in source, ( + "get_settings() defaults must use the named image generation API version constant" + ) + assert f"'azure_openai_image_gen_api_version': '{PREVIOUS_DEFAULT}'" not in source + assert "'azure_apim_image_gen_api_version': ''," in source, ( + "The APIM image generation API version must keep having no default" + ) + + +def test_previous_default_is_upgraded(): + """A stored value equal to the previous default was never customized, so it is upgraded.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = { + "azure_openai_image_gen_api_version": PREVIOUS_DEFAULT, + "azure_apim_image_gen_api_version": "", + "unrelated_key": "left alone", + } + + changed = migrate(settings) + + assert changed is True + assert settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert settings["azure_apim_image_gen_api_version"] == "" + assert settings["unrelated_key"] == "left alone" + + padded = {"azure_openai_image_gen_api_version": f" {PREVIOUS_DEFAULT} "} + assert migrate(padded) is True and padded["azure_openai_image_gen_api_version"] == NEW_DEFAULT + + +@pytest.mark.parametrize("stored_version", ["", " ", None]) +def test_blank_values_are_upgraded(stored_version): + """A blank stored value would send no API version, so it becomes the new default.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {"azure_openai_image_gen_api_version": stored_version} + + changed = migrate(settings) + + assert changed is True + assert settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + + +def test_missing_value_is_filled(): + """A document without the key gets the new default rather than nothing.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {} + + changed = migrate(settings) + + assert changed is True + assert settings == {"azure_openai_image_gen_api_version": NEW_DEFAULT} + + +@pytest.mark.parametrize("custom_version", ["2025-03-01-preview", "2024-10-21", "2024-05-01-preview", NEW_DEFAULT]) +def test_custom_values_are_left_untouched(custom_version): + """Any other value is an admin's deliberate choice, and an up-to-date value needs no write.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {"azure_openai_image_gen_api_version": custom_version} + + changed = migrate(settings) + + assert changed is False + assert settings["azure_openai_image_gen_api_version"] == custom_version + + +def test_apim_version_is_never_changed(): + """APIM gateways define their own API version, so it is never migrated.""" + migrate = load_migration()[MIGRATION_FUNCTION] + for apim_version in (PREVIOUS_DEFAULT, "", " ", "2024-02-01"): + settings = { + "azure_openai_image_gen_api_version": PREVIOUS_DEFAULT, + "azure_apim_image_gen_api_version": apim_version, + } + migrate(settings) + assert settings["azure_apim_image_gen_api_version"] == apim_version, ( + f"The APIM API version {apim_version!r} must not be migrated" + ) + + apim_only = {"azure_openai_image_gen_api_version": "2025-03-01-preview", "azure_apim_image_gen_api_version": PREVIOUS_DEFAULT} + assert migrate(apim_only) is False + assert apim_only["azure_apim_image_gen_api_version"] == PREVIOUS_DEFAULT + + +def test_unexpected_input_is_handled_safely(): + """Non-dict settings and non-string values must not raise or be rewritten.""" + migrate = load_migration()[MIGRATION_FUNCTION] + assert migrate(None) is False + assert migrate("not-a-dict") is False + + settings = {"azure_openai_image_gen_api_version": 20250401} + assert migrate(settings) is False + assert settings["azure_openai_image_gen_api_version"] == 20250401 + + +def test_migration_is_wired_into_every_settings_load(): + """get_settings() must run the migration inside normalize_loaded_settings().""" + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + loader = _function_node(_function_node(tree, "get_settings"), "normalize_loaded_settings") + calls = [ + node for node in ast.walk(loader) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == MIGRATION_FUNCTION + ] + assert len(calls) == 1, f"normalize_loaded_settings() must call {MIGRATION_FUNCTION}() once" + assert [ast.unparse(arg) for arg in calls[0].args] == ["merged"], "The migration must run on the merged settings" + + +def test_docs_and_example_artifacts_show_the_new_default(): + """Admin docs and example settings must match the shipped image generation default.""" + ai_models_doc = (ROOT / "docs" / "admin" / "ai-models.md").read_text(encoding="utf-8") + image_row = next(line for line in ai_models_doc.splitlines() if "`azure_openai_image_gen_api_version`" in line) + assert f"| {NEW_DEFAULT} |" in image_row, f"The Image Gen API Version row must show {NEW_DEFAULT}: {image_row}" + apim_row = next(line for line in ai_models_doc.splitlines() if "`azure_apim_image_gen_api_version`" in line) + assert "| Empty |" in apim_row, "The APIM image generation API version must still have no default" + + cosmos_example = json.loads((ROOT / "artifacts" / "cosmos_examples" / "cosmos-settings-example.json").read_text(encoding="utf-8")) + assert cosmos_example["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert cosmos_example["azure_openai_gpt_api_version"] == PREVIOUS_DEFAULT, "Only the image generation entry changes" + assert cosmos_example["azure_openai_embedding_api_version"] == PREVIOUS_DEFAULT, "Only the image generation entry changes" + + seeder_settings = json.loads( + (ROOT / "application" / "external_apps" / "databaseseeder" / "artifacts" / "admin_settings.json").read_text(encoding="utf-8") + ) + assert seeder_settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert seeder_settings["azure_apim_image_gen_api_version"] == "" + + +CONNECTION_MIGRATION_FUNCTION = "normalize_image_connection_api_version_settings" + + +def load_connection_migration(): + """Load the React V2 connection-profile migration from source, like load_migration().""" + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + wanted_constants = {DEFAULT_CONSTANT, PREVIOUS_DEFAULT_CONSTANT} + selected_nodes = [ + node for node in tree.body + if (isinstance(node, ast.FunctionDef) and node.name == CONNECTION_MIGRATION_FUNCTION) + or ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in wanted_constants + ) + ] + assert len(selected_nodes) == 3, f"Expected {CONNECTION_MIGRATION_FUNCTION}() at module level" + namespace = {"IMAGE_GENERATION_CAPABILITY": "image_generation"} + exec(compile(ast.Module(body=selected_nodes, type_ignores=[]), str(SETTINGS_FILE), "exec"), namespace) + return namespace[CONNECTION_MIGRATION_FUNCTION] + + +def _image_endpoint(endpoint_id, api_version, is_apim=False): + profile = {"is_apim": is_apim} + if api_version is not None: + profile["api_version"] = api_version + return { + "id": endpoint_id, + "provider": "aoai", + "connection": { + "endpoint": f"https://{endpoint_id}.invalid", + "api_version": "2024-05-01-preview", + "operation_settings": {"image_generation": profile}, + }, + "models": [], + } + + +def test_imported_image_connection_profiles_are_upgraded(): + """V2: a direct image profile still holding the previous default is upgraded, nothing else.""" + migrate = load_connection_migration() + settings = { + "model_endpoints": [ + _image_endpoint("imported-direct", PREVIOUS_DEFAULT), + _image_endpoint("padded", f" {PREVIOUS_DEFAULT} "), + _image_endpoint("gateway", PREVIOUS_DEFAULT, is_apim=True), + _image_endpoint("custom", "2025-03-01-preview"), + _image_endpoint("blank", ""), + _image_endpoint("unset", None), + {"id": "chat-only", "connection": {"api_version": PREVIOUS_DEFAULT}}, + "not-an-endpoint", + {"id": "bad-profiles", "connection": {"operation_settings": ["image_generation"]}}, + ], + } + + assert migrate(settings) is True + profiles = { + endpoint["id"]: endpoint["connection"].get("operation_settings", {}) + for endpoint in settings["model_endpoints"] + if isinstance(endpoint, dict) + } + assert profiles["imported-direct"]["image_generation"]["api_version"] == NEW_DEFAULT + assert profiles["padded"]["image_generation"]["api_version"] == NEW_DEFAULT + assert profiles["gateway"]["image_generation"]["api_version"] == PREVIOUS_DEFAULT, "APIM must not change" + assert profiles["custom"]["image_generation"]["api_version"] == "2025-03-01-preview" + assert profiles["blank"]["image_generation"]["api_version"] == "" + assert "api_version" not in profiles["unset"]["image_generation"] + chat_only = next(endpoint for endpoint in settings["model_endpoints"] if isinstance(endpoint, dict) and endpoint["id"] == "chat-only") + assert chat_only["connection"]["api_version"] == PREVIOUS_DEFAULT, "A chat API version is never an image version" + + assert migrate(settings) is False, "A second load must not report a change" + assert migrate({}) is False and migrate({"model_endpoints": "x"}) is False and migrate(None) is False + + +def test_connection_profile_migration_is_wired_into_every_settings_load(): + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + loader = _function_node(_function_node(tree, "get_settings"), "normalize_loaded_settings") + calls = [ + node for node in ast.walk(loader) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == CONNECTION_MIGRATION_FUNCTION + ] + assert len(calls) == 1 and [ast.unparse(arg) for arg in calls[0].args] == ["merged"] + + +def test_v2_legacy_images_fallback_and_admin_field_use_the_new_default(): + """V2: legacy Images settings with no version, and the V2 admin field, use the new default.""" + route_source = (APP_ROOT / "functions_image_api_route.py").read_text(encoding="utf-8") + assert f"DEFAULT_IMAGES_API_VERSION = '{NEW_DEFAULT}'" in route_source + assert "LEGACY_DEFAULT_IMAGES_API_VERSION = DEFAULT_IMAGES_API_VERSION" in route_source + old_default_assignments = [ + node for node in ast.parse(route_source).body + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant) and node.value.value == PREVIOUS_DEFAULT + ] + assert not old_default_assignments, "No module constant may keep the old default" + + fields_tree = ast.parse((APP_ROOT / "admin_settings_fields.py").read_text(encoding="utf-8")) + field_defaults = [ + node for node in ast.walk(fields_tree) + if isinstance(node, ast.Dict) + and any(isinstance(key, ast.Constant) and key.value == "key" for key in node.keys) + and any( + isinstance(value, ast.Constant) and value.value == "azure_openai_image_gen_api_version" + for value in node.values + ) + ] + assert len(field_defaults) == 1, "Expected one V2 admin field for the image generation API version" + defaults = { + key.value: value.value + for key, value in zip(field_defaults[0].keys, field_defaults[0].values) + if isinstance(key, ast.Constant) and isinstance(value, ast.Constant) + } + assert defaults.get("default") == NEW_DEFAULT, defaults + + +def _run_offline_probe(): + # Real application imports must occur inside the external-I/O bootstrap seam. + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline: + import functions_settings as settings_module # noqa: PLC0415 + _require(not offline.network_attempts, "Settings imports attempted network access.") + container = settings_module.cosmos_settings_container + + for stored_version, apim_version, expected_version in ( + (PREVIOUS_DEFAULT, PREVIOUS_DEFAULT, NEW_DEFAULT), + ("", "", NEW_DEFAULT), + (" ", "2024-02-01", NEW_DEFAULT), + ("2025-03-01-preview", "", "2025-03-01-preview"), + ): + container.upsert_item({ + "id": "app_settings", + "azure_openai_image_gen_api_version": stored_version, + "azure_apim_image_gen_api_version": apim_version, + }) + + loaded = settings_module.get_settings() + _require(isinstance(loaded, dict), "Settings must load.") + _require(loaded["azure_openai_image_gen_api_version"] == expected_version, + f"Loaded {loaded['azure_openai_image_gen_api_version']!r} for stored {stored_version!r}.") + _require(loaded["azure_apim_image_gen_api_version"] == apim_version, + f"The APIM version {apim_version!r} must not be migrated.") + + persisted = container.read_item(item="app_settings", partition_key="app_settings") + _require(persisted["azure_openai_image_gen_api_version"] == expected_version, + f"The migrated value must be persisted for stored {stored_version!r}.") + _require(persisted["azure_apim_image_gen_api_version"] == apim_version, + "The persisted APIM version must be unchanged.") + + _require(settings_module.get_settings()["azure_openai_image_gen_api_version"] == "2025-03-01-preview", + "A custom value must survive repeated loads.") + + # React V2: an imported direct image connection is upgraded and persisted; APIM is not. + container.upsert_item({ + "id": "app_settings", + "azure_openai_image_gen_api_version": NEW_DEFAULT, + "model_endpoints": [ + _image_endpoint("imported-direct", PREVIOUS_DEFAULT), + _image_endpoint("gateway", PREVIOUS_DEFAULT, is_apim=True), + ], + }) + for label, document in ( + ("loaded", settings_module.get_settings()), + ("persisted", container.read_item(item="app_settings", partition_key="app_settings")), + ): + versions = { + endpoint["id"]: endpoint["connection"]["operation_settings"]["image_generation"].get("api_version") + for endpoint in document["model_endpoints"] + } + _require(versions.get("imported-direct") == NEW_DEFAULT, + f"The {label} imported image profile must use the new default: {versions}.") + _require(versions.get("gateway") == PREVIOUS_DEFAULT, + f"The {label} APIM image profile must be unchanged: {versions}.") + _require(not offline.network_attempts, "Settings loads attempted network access.") + + print("Image generation API version load migration checks passed.") + + +@pytest.mark.parametrize("optimized", (False, True)) +def test_settings_load_migrates_and_persists_the_api_version(optimized): + """Run the real get_settings() load path in a fresh process with external I/O blocked.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(APP_ROOT), str(TEST_ROOT))) + env["PYTHONIOENCODING"] = "utf-8" + command = [sys.executable] + if optimized: + command.append("-O") + command.extend((str(Path(__file__).resolve()), "--offline-probe")) + result = subprocess.run( + command, cwd=ROOT, env=env, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=300, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_version_is_at_least_implementation_version(): + """The app version must be at or beyond the version this fix shipped in.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + + +if __name__ == "__main__": + if "--offline-probe" in sys.argv: + _run_offline_probe() + else: + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_image_generation_sdk_http.py b/functional_tests/test_image_generation_sdk_http.py index a0b35e786..c7aca8c38 100644 --- a/functional_tests/test_image_generation_sdk_http.py +++ b/functional_tests/test_image_generation_sdk_http.py @@ -1,8 +1,8 @@ # test_image_generation_sdk_http.py """ Functional tests for image URLs, authentication, payloads, and output with the real OpenAI SDK. -Version: 0.261.107 -Implemented in: 0.261.105 +Version: 0.261.210 +Implemented in: 0.261.105; legacy Images fallback version updated in 0.261.210 Uses the application's pinned openai==1.109.1 client and httpx.MockTransport. Only HTTP is mocked: request construction, auth_headers, multipart encoding, response parsing, @@ -454,7 +454,8 @@ def test_unmigrated_legacy_images_keep_their_sdk_route_but_gpt_is_rejected(self) self.assertEqual(self.requests[-1].headers["api-key"], "legacy-key") settings.pop("azure_openai_image_gen_api_version") generation.request_generated_image_source(settings, "Legacy image without a saved version") - self.assertEqual(dict(self.requests[-1].url.params), {"api-version": "2024-12-01-preview"}) + # Since 0.261.210 a legacy route with no saved version uses the gpt-image default. + self.assertEqual(dict(self.requests[-1].url.params), {"api-version": "2025-04-01-preview"}) settings["image_gen_model"]["selected"] = [{ "deploymentName": "legacy-gpt", "modelName": "gpt-5.6-terra", }] From 3d77cbe14aae951f9fd4bd88247ea6a5b0e24bf0 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 13:54:01 -0400 Subject: [PATCH 4/6] Add React V2 release notes for the image indexing, upload extraction and image API version fixes Refs #1583 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...SCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md | 20 ++++++++++++- docs/explanation/release_notes.md | 29 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md index 9004d754a..c5ca7a1e6 100644 --- a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md +++ b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md @@ -1,9 +1,27 @@ # Image Description Indexing Without OCR Text Fix (v0.261.047) -Fixed in version: **0.261.047** +Fixed in version: **0.261.047** (Development) and **0.261.210** (React V2) Issue: [#1583](https://github.com/microsoft/simplechat/issues/1583) +## React V2 + +The same fix ships on the React V2 branch in **0.261.210**, with these differences, because +React V2 enforces model capabilities and embedding limits that Development does not: + +- The shared AI connection client helper finds the model and re-checks its chat capability + before resolving any Key Vault secret, as React V2 metadata extraction always did. A missing + model raises the same `LookupError`, and a disabled model is refused by the shared capability + check with "The selected model is not available for chat. Choose a compatible model." +- Vision analysis re-checks image support against the resolved connection's model record. The + legacy GPT path keeps its existing capability checks. +- React V2 refuses a chunk that is too large for a non-legacy embedding profile instead of + clamping it. When the appended vision block would push a page that fits over that limit, the + page's OCR text is embedded on its own, as before, and the stored chunk text still includes + the block. +- V2 chat shows what was extracted from an uploaded image. See + [V2 Chat Upload Extraction and Image Card Fix](V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md). + ## Issue Description A user uploaded two nearly identical grayscale photos of an empty lab floor. Neither photo has diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index cc1262e02..f54b7c721 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,35 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.210)** + +#### Bug Fixes + +* **Images Without OCR Text Are Now Indexed From Their Vision Description** + * Fixed text-free images, such as a photo of an empty room, ending with no search chunk, no title and nothing chat could use, even though the AI vision model had described them. When OCR finds no text, the vision description is now indexed as the image's chunk, so the image is searchable and final metadata extraction runs as usual. + * Document Intelligence Layout output that is only page-number or page-break annotations, such as a stray `` read from floor tape, no longer counts as image text. Header and footer text still does. Failed or empty vision analyses are never indexed. + * Image chunks are now embedded from the same text that is stored, including the vision description, instead of from the OCR text alone. When the description would push a page past the limit of a non-legacy embedding model, the page's OCR text is embedded on its own, as before. + * With Enhanced extraction, a standalone image that the Content Understanding document analyzer returns nothing for is now described by the Content Understanding image analyzer before falling back to Document Intelligence Layout, for workspace uploads and images uploaded straight into a chat. The image analyzer's `Summary` field is now read, which also replaces the `![image](pages/1)` placeholder that images embedded in DOCX and PPTX files were indexed with. + * The multi-modal vision model is now called through the AI connection that hosts it, instead of always using the primary GPT connection, which returned 404 for a model deployed on another connection. The model's chat and image support are checked on that connection first, and Test Vision Analysis uses the same connection as ingestion. + * (Ref: #1583, `functions_documents.py`, `functions_content.py`, `functions_content_understanding.py`, `functions_model_endpoint_types.py`, `route_backend_settings.py`, `route_frontend_chats.py`, [Image Description Indexing Without OCR Text Fix](fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md)) + +* **Uploaded Files Show What Was Extracted Instead Of "File content not found"** + * Opening a file uploaded into a conversation, including an image, showed "File content not found" whenever the upload was stored as a workspace document. The preview now shows what ingestion extracted: the engine that read the file and why, the AI vision analysis, the title and summary, and the indexed text that search and chat read. It explains when a file is still processing or when nothing from it was indexed. The classic file popup shows the same text. + * A cited image in V2 chat now shows **What was extracted from this image**, the passage the assistant was given, beneath the picture. + * (Ref: #1583, `route_backend_documents.py`, `functions_chat_upload_extraction.py`, `ChatUploadExtraction.tsx`, `ChatFilePreview.tsx`, `EnhancedCitationViewer.tsx`, [V2 Chat Upload Extraction and Image Card Fix](fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md)) + +* **Image Generation Uses The gpt-image API Version By Default** + * The default Azure OpenAI image generation API version is now `2025-04-01-preview`, the newest dated Azure OpenAI API version and the one documented for gpt-image-1, gpt-image-1.5 and gpt-image-2. The previous `2024-12-01-preview` default predates gpt-image support. + * A saved legacy image version or AI connection image profile still holding the previous default is upgraded automatically when settings load, and a blank legacy value is filled in. Any other saved version is kept, and APIM image versions are never changed because the gateway defines them. Unmigrated legacy Images settings with no version also use the new default. + * (Ref: `functions_settings.py`, `functions_image_api_route.py`, `admin_settings_fields.py`, `azure_openai_image_gen_api_version`, [Image Generation API Version Default Fix](fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md)) + +#### User Interface Enhancements + +* **Uploaded Images Match Generated Images In V2 Chat** + * An image uploaded into a V2 conversation is now drawn at its own size in a compact card, the same way a generated image is, instead of filling the message column with the picture letterboxed. + * Its actions appear when you point at or tab to the image. **Show sources** opens a panel with what was extracted from it, alongside **Message details**, **Open file preview**, **Use as reference** and **Edit**. + * (Ref: #1583, `MessageList.tsx`, `MessageInspector.tsx`, `MessageActions.tsx`, [Chat controls](../reference/chat-controls.md)) + ### **(v0.261.209)** #### Bug Fixes From 5a3c93c3c04403f689553d3d559d6b26233bfbcd Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 14:23:53 -0400 Subject: [PATCH 5/6] Address review findings on the React V2 image fixes - An image whose only text is its vision description, or a page the appended description pushes past the embedding budget, now embeds the stored text cut to the budget instead of failing ingestion under a non-legacy embedding profile. The stored chunk keeps the whole description. - A chat upload whose ingestion failed is reported as failed rather than still processing, in both the file content response and the V2 extraction panel. - An unmigrated legacy APIM image route with no API version keeps sending 2024-12-01-preview; only the direct route falls back to 2025-04-01-preview. Refs #1583 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../functions_chat_upload_extraction.py | 22 ++++++++++---- application/single_app/functions_documents.py | 21 ++++++++++---- .../single_app/functions_image_api_route.py | 18 +++++++----- .../components/chat/ChatUploadExtraction.tsx | 10 ++++--- docs/admin/ai-models.md | 6 ++-- .../IMAGE_GENERATION_RESPONSES_MODELS.md | 3 +- ...SCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md | 7 +++-- ...MAGE_GENERATION_API_VERSION_DEFAULT_FIX.md | 7 +++-- ...AT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md | 4 ++- docs/explanation/release_notes.md | 6 ++-- .../test_chat_upload_extraction_details.py | 13 +++++++++ ...e_description_indexing_without_ocr_text.py | 14 +++++++-- ...st_image_generation_api_version_default.py | 29 +++++++++++++++++-- ui_tests/test_v2_chat_image_uploads.py | 20 +++++++++++++ 14 files changed, 142 insertions(+), 38 deletions(-) diff --git a/application/single_app/functions_chat_upload_extraction.py b/application/single_app/functions_chat_upload_extraction.py index 672f6232f..450154aba 100644 --- a/application/single_app/functions_chat_upload_extraction.py +++ b/application/single_app/functions_chat_upload_extraction.py @@ -13,6 +13,9 @@ WORKSPACE_UPLOAD_FILE_CONTENT_SOURCE = "workspace" NO_EXTRACTED_CONTENT_MESSAGE = "No extracted content is available for this file yet." STILL_PROCESSING_MESSAGE = "This file is still being processed. Try again when it finishes." +PROCESSING_FAILED_MESSAGE = ( + "Processing this file failed, so nothing was extracted. Reprocess or re-upload it to try again." +) def _clean_text(value): @@ -88,8 +91,15 @@ def _percentage_complete(document): return None +def is_workspace_upload_failed(document): + """Return whether ingestion stopped with an error, which is final rather than in progress.""" + return _clean_text(document.get("status")).lower().startswith("error") + + def is_workspace_upload_processing(document): """Return whether the linked document has not finished ingestion yet.""" + if is_workspace_upload_failed(document): + return False percentage = _percentage_complete(document) if percentage is not None: return percentage < 100 @@ -137,11 +147,13 @@ def build_workspace_upload_file_content(filename, document, chunks): body = indexed_text or render_vision_analysis_text(details["vision_analysis"]) if not body: - message = ( - STILL_PROCESSING_MESSAGE - if is_workspace_upload_processing(document if isinstance(document, dict) else {}) - else NO_EXTRACTED_CONTENT_MESSAGE - ) + source = document if isinstance(document, dict) else {} + if is_workspace_upload_failed(source): + message = PROCESSING_FAILED_MESSAGE + elif is_workspace_upload_processing(source): + message = STILL_PROCESSING_MESSAGE + else: + message = NO_EXTRACTED_CONTENT_MESSAGE return {"error": message, "workspace_document": details}, 404 return { diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index 27e8ce7f9..e1c47a4de 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -4031,17 +4031,28 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, embedding_input = enhanced_chunk_text max_embedding_characters = get_embedding_safe_chunk_characters() - # The vision block is added to the page text. When that alone pushes a page that fits - # over the budget, embed the page text by itself, as before the block was embedded, - # rather than refusing a chunk the selected embedding model could always take. + # Processors bound the page text, not the vision block appended to it. When the block + # alone pushes a page that fits over the budget -- including an image whose only text + # is its description -- embed the leading part of the stored text instead of refusing + # a chunk the page itself never made too large. The full text is still stored below. page_text = page_text_content or '' if ( vision_text and len(embedding_input) > max_embedding_characters - and page_text.strip() and len(page_text) <= max_embedding_characters ): - embedding_input = page_text + log_event( + "[SAVE_CHUNKS] The vision description was clamped for embedding only.", + extra={ + "document_id": document_id, + "page_number": page_number, + "original_characters": len(embedding_input), + "clamped_characters": max_embedding_characters, + }, + level=logging.INFO, + debug_only=True, + ) + embedding_input = embedding_input[:max_embedding_characters] # Last-resort guard. Every processor bounds its own chunks, so reaching this means content # tokenized far worse than estimated. Splitting is not an option here because chunk ids are diff --git a/application/single_app/functions_image_api_route.py b/application/single_app/functions_image_api_route.py index 28f5fecad..75b4ea927 100644 --- a/application/single_app/functions_image_api_route.py +++ b/application/single_app/functions_image_api_route.py @@ -41,9 +41,12 @@ # New shared connections support both Images generations and edits without inheriting # a chat API version. Explicit imported image profiles always take precedence. DEFAULT_IMAGES_API_VERSION = '2025-04-01-preview' -# Unmigrated legacy Images settings with no version use the same default. The previous +# Unmigrated legacy direct Images settings with no version use the same default. The previous # fallback, 2024-12-01-preview, predates gpt-image model support. LEGACY_DEFAULT_IMAGES_API_VERSION = DEFAULT_IMAGES_API_VERSION +# An unmigrated legacy APIM image route with no version keeps the fallback it has always sent, +# because the gateway, not SimpleChat, decides which API versions it accepts. +LEGACY_APIM_DEFAULT_IMAGES_API_VERSION = '2024-12-01-preview' # What the image_generation tool call answers with when no format is stated. DEFAULT_RESPONSES_IMAGE_FORMAT = 'png' @@ -229,12 +232,13 @@ def resolve_image_generation_api_version(settings): return 'preview' if image_settings_use_connections(settings): return resolve_image_binding_api_version(resolve_shared_image_binding(settings)) - key = ( - 'azure_apim_image_gen_api_version' - if settings.get('enable_image_gen_apim') - else 'azure_openai_image_gen_api_version' - ) - return str(settings.get(key) or LEGACY_DEFAULT_IMAGES_API_VERSION).strip() + if settings.get('enable_image_gen_apim'): + return str( + settings.get('azure_apim_image_gen_api_version') or LEGACY_APIM_DEFAULT_IMAGES_API_VERSION + ).strip() + return str( + settings.get('azure_openai_image_gen_api_version') or LEGACY_DEFAULT_IMAGES_API_VERSION + ).strip() def resolve_selected_image_capability(settings): diff --git a/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx b/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx index c34f0509e..411ca0d5f 100644 --- a/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx +++ b/application/v2_ui/src/components/chat/ChatUploadExtraction.tsx @@ -173,10 +173,12 @@ export function ChatUploadExtraction({ data }: { data: ChatFileContent }) { ) : (

- {typeof details.percentage_complete === 'number' && - details.percentage_complete < 100 - ? 'Still processing. The indexed text appears here once ingestion finishes.' - : 'Nothing from this file was indexed for search, so chat cannot cite it. Reprocess or re-upload the file to index it.'} + {String(details.status ?? '').trim().toLowerCase().startsWith('error') + ? 'Processing this file failed, so nothing from it was indexed for search. Reprocess or re-upload the file to index it.' + : typeof details.percentage_complete === 'number' && + details.percentage_complete < 100 + ? 'Still processing. The indexed text appears here once ingestion finishes.' + : 'Nothing from this file was indexed for search, so chat cannot cite it. Reprocess or re-upload the file to index it.'}

)}
diff --git a/docs/admin/ai-models.md b/docs/admin/ai-models.md index 840809ef8..3cbd9d60b 100644 --- a/docs/admin/ai-models.md +++ b/docs/admin/ai-models.md @@ -400,8 +400,10 @@ New shared Images connections without an image-specific API version use including older imported versions, are preserved, except the previous default `2024-12-01-preview`, which predates gpt-image support: since **0.261.210** a direct (non-APIM) image profile or legacy image setting still holding that exact value is -upgraded to `2025-04-01-preview` when settings load. Unmigrated legacy Images settings -with no version configured also use `2025-04-01-preview`. +upgraded to `2025-04-01-preview` when settings load. Unmigrated legacy direct Images +settings with no version configured also use `2025-04-01-preview`. An unmigrated legacy +APIM route with no version keeps sending `2024-12-01-preview`, because the gateway decides +which versions it accepts. These versions do not come from the chat connection's API version or legacy root settings after shared selection. Direct OpenAI uses `/v1` without an Azure diff --git a/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md b/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md index 29fc4318e..8dd9d140a 100644 --- a/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md +++ b/docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md @@ -80,7 +80,8 @@ Images and Responses have separate version contracts: | --- | --- | | Shared Images model with no image-specific version, including a missing operation profile | `2025-04-01-preview` for Images generations and edits | | Shared Images model with an explicit `connection.operation_settings.image_generation.api_version` | The stored version, including an older imported version. Since 0.261.210 a direct profile still holding the previous default `2024-12-01-preview` is upgraded to `2025-04-01-preview` when settings load. | -| Unmigrated legacy Images route without a configured image API version | `2025-04-01-preview` (`2024-12-01-preview` before 0.261.210) | +| Unmigrated legacy direct Images route without a configured image API version | `2025-04-01-preview` (`2024-12-01-preview` before 0.261.210) | +| Unmigrated legacy APIM Images route without a configured image API version | `2024-12-01-preview`; the gateway decides which versions it accepts | | Direct OpenAI Images or Responses | OpenAI `/v1`, without an Azure version query | | MAI Image | `/mai/v1`, without an Azure OpenAI version query | | Foundry FLUX | `api-version=preview`; native per-model paths or the documented v1 Images-compatible Kontext paths | diff --git a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md index c5ca7a1e6..04127d82f 100644 --- a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md +++ b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md @@ -16,9 +16,10 @@ React V2 enforces model capabilities and embedding limits that Development does - Vision analysis re-checks image support against the resolved connection's model record. The legacy GPT path keeps its existing capability checks. - React V2 refuses a chunk that is too large for a non-legacy embedding profile instead of - clamping it. When the appended vision block would push a page that fits over that limit, the - page's OCR text is embedded on its own, as before, and the stored chunk text still includes - the block. + clamping it. Page text is bounded by its processor, but the appended vision block is not, so + when the block pushes a page that fits over that limit, including an image whose only text is + its description, the embedding input is cut to the limit and the stored chunk text keeps the + whole block. A page whose own text is too large is still refused, as before. - V2 chat shows what was extracted from an uploaded image. See [V2 Chat Upload Extraction and Image Card Fix](V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md). diff --git a/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md index 7cbb364f1..af72536c0 100644 --- a/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md +++ b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md @@ -14,8 +14,11 @@ places in **0.261.210**: profiles, other versions and blank versions are left alone. A blank version already falls back to `2025-04-01-preview` at request time. - `LEGACY_DEFAULT_IMAGES_API_VERSION` in `functions_image_api_route.py`, the fallback for - unmigrated legacy Images settings with no version, now equals `DEFAULT_IMAGES_API_VERSION` - (`2025-04-01-preview`). + unmigrated legacy direct Images settings with no version, now equals + `DEFAULT_IMAGES_API_VERSION` (`2025-04-01-preview`). An unmigrated legacy APIM route with no + version keeps its `2024-12-01-preview` fallback, now named + `LEGACY_APIM_DEFAULT_IMAGES_API_VERSION`, because the gateway decides which versions it + accepts. - The V2 admin field for `azure_openai_image_gen_api_version` in `admin_settings_fields.py` shows the new default, and `docs/admin/ai-models.md` and `docs/explanation/features/IMAGE_GENERATION_RESPONSES_MODELS.md` describe the upgrade. diff --git a/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md b/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md index d9a1c14b4..7c33d6736 100644 --- a/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md +++ b/docs/explanation/fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md @@ -79,7 +79,9 @@ For a workspace-backed upload, `/api/get_file_content` returns: `file_content` is always readable text, so the classic file popup, which renders only `file_content`, also works for these uploads now. When there is nothing to show, the route returns 404 with "This file is still being processed. Try again when it finishes." while -ingestion is running, or "No extracted content is available for this file yet." afterwards. +ingestion is running, "Processing this file failed, so nothing was extracted. Reprocess or +re-upload it to try again." when the document's status is an error, or "No extracted content +is available for this file yet." otherwise. Tabular uploads and screened documents keep their existing paths. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index f54b7c721..20b1cef07 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -9,19 +9,19 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Ver * **Images Without OCR Text Are Now Indexed From Their Vision Description** * Fixed text-free images, such as a photo of an empty room, ending with no search chunk, no title and nothing chat could use, even though the AI vision model had described them. When OCR finds no text, the vision description is now indexed as the image's chunk, so the image is searchable and final metadata extraction runs as usual. * Document Intelligence Layout output that is only page-number or page-break annotations, such as a stray `` read from floor tape, no longer counts as image text. Header and footer text still does. Failed or empty vision analyses are never indexed. - * Image chunks are now embedded from the same text that is stored, including the vision description, instead of from the OCR text alone. When the description would push a page past the limit of a non-legacy embedding model, the page's OCR text is embedded on its own, as before. + * Image chunks are now embedded from the same text that is stored, including the vision description, instead of from the OCR text alone. When the description pushes a page that fits past the limit of a non-legacy embedding model, including an image whose only text is its description, the embedding input is cut to the limit and the stored chunk keeps the whole description. * With Enhanced extraction, a standalone image that the Content Understanding document analyzer returns nothing for is now described by the Content Understanding image analyzer before falling back to Document Intelligence Layout, for workspace uploads and images uploaded straight into a chat. The image analyzer's `Summary` field is now read, which also replaces the `![image](pages/1)` placeholder that images embedded in DOCX and PPTX files were indexed with. * The multi-modal vision model is now called through the AI connection that hosts it, instead of always using the primary GPT connection, which returned 404 for a model deployed on another connection. The model's chat and image support are checked on that connection first, and Test Vision Analysis uses the same connection as ingestion. * (Ref: #1583, `functions_documents.py`, `functions_content.py`, `functions_content_understanding.py`, `functions_model_endpoint_types.py`, `route_backend_settings.py`, `route_frontend_chats.py`, [Image Description Indexing Without OCR Text Fix](fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md)) * **Uploaded Files Show What Was Extracted Instead Of "File content not found"** - * Opening a file uploaded into a conversation, including an image, showed "File content not found" whenever the upload was stored as a workspace document. The preview now shows what ingestion extracted: the engine that read the file and why, the AI vision analysis, the title and summary, and the indexed text that search and chat read. It explains when a file is still processing or when nothing from it was indexed. The classic file popup shows the same text. + * Opening a file uploaded into a conversation, including an image, showed "File content not found" whenever the upload was stored as a workspace document. The preview now shows what ingestion extracted: the engine that read the file and why, the AI vision analysis, the title and summary, and the indexed text that search and chat read. It explains when a file is still processing, when processing failed, or when nothing from it was indexed. The classic file popup shows the same text. * A cited image in V2 chat now shows **What was extracted from this image**, the passage the assistant was given, beneath the picture. * (Ref: #1583, `route_backend_documents.py`, `functions_chat_upload_extraction.py`, `ChatUploadExtraction.tsx`, `ChatFilePreview.tsx`, `EnhancedCitationViewer.tsx`, [V2 Chat Upload Extraction and Image Card Fix](fixes/V2_CHAT_UPLOAD_EXTRACTION_AND_IMAGE_CARD_FIX.md)) * **Image Generation Uses The gpt-image API Version By Default** * The default Azure OpenAI image generation API version is now `2025-04-01-preview`, the newest dated Azure OpenAI API version and the one documented for gpt-image-1, gpt-image-1.5 and gpt-image-2. The previous `2024-12-01-preview` default predates gpt-image support. - * A saved legacy image version or AI connection image profile still holding the previous default is upgraded automatically when settings load, and a blank legacy value is filled in. Any other saved version is kept, and APIM image versions are never changed because the gateway defines them. Unmigrated legacy Images settings with no version also use the new default. + * A saved legacy image version or AI connection image profile still holding the previous default is upgraded automatically when settings load, and a blank legacy value is filled in. Any other saved version is kept, and APIM image versions are never changed because the gateway defines them. Unmigrated legacy direct Images settings with no version also use the new default; an unmigrated APIM route with no version keeps sending `2024-12-01-preview`. * (Ref: `functions_settings.py`, `functions_image_api_route.py`, `admin_settings_fields.py`, `azure_openai_image_gen_api_version`, [Image Generation API Version Default Fix](fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md)) #### User Interface Enhancements diff --git a/functional_tests/test_chat_upload_extraction_details.py b/functional_tests/test_chat_upload_extraction_details.py index 4fd7d1f21..86eaa3417 100644 --- a/functional_tests/test_chat_upload_extraction_details.py +++ b/functional_tests/test_chat_upload_extraction_details.py @@ -147,6 +147,19 @@ def test_nothing_to_show_explains_processing_or_absence(): assert status == 404 assert payload["error"] == extraction.STILL_PROCESSING_MESSAGE + # A failed document is final: ingestion saves percentage 0 with an Error status. + failed = {"id": "doc-failed", "status": "Error: Processing failed: The chunk is too large", "percentage_complete": 0} + assert extraction.is_workspace_upload_processing(failed) is False + payload, status = extraction.build_workspace_upload_file_content("lab.png", failed, []) + assert status == 404 + assert payload["error"] == extraction.PROCESSING_FAILED_MESSAGE + assert payload["workspace_document"]["status"].startswith("Error:") + + # A failed document that still stored its vision analysis shows it rather than an error. + failed_with_vision = {**failed, "vision_analysis": VISION_ANALYSIS} + payload, status = extraction.build_workspace_upload_file_content("lab.png", failed_with_vision, []) + assert status == 200 and payload["indexed_text_available"] is False + def _load_route_helper(): """Compile the route's helper on its own, with its collaborators supplied by the test.""" diff --git a/functional_tests/test_image_description_indexing_without_ocr_text.py b/functional_tests/test_image_description_indexing_without_ocr_text.py index d272a2481..40f5c7e01 100644 --- a/functional_tests/test_image_description_indexing_without_ocr_text.py +++ b/functional_tests/test_image_description_indexing_without_ocr_text.py @@ -541,12 +541,20 @@ def check_save_chunks_embeds_the_stored_text(documents): _require(uploaded[0]["chunk_text"] == long_text + block, "The stored chunk text must never be clamped.") # React V2: a non-legacy embedding profile refuses an oversized chunk. Appending the vision - # block must not turn a page that fits into a refused one, so that page embeds its OCR text. + # block must not turn a page that fits into a refused one, so the embedding input is cut to + # the budget while the stored chunk keeps the full text. _, embedded, uploaded = _run_save_chunks( documents, "EXIT 12", VISION_ANALYSIS, max_characters=60, legacy_embedding=False, ) - _require(embedded == ["EXIT 12"], "A page that fits must embed its OCR text when the vision block does not.") - _require(uploaded[0]["chunk_text"] == "EXIT 12" + block, "The stored text must still carry the vision block.") + _require(embedded == [("EXIT 12" + block)[:60]], "A page that fits must embed its clamped stored text.") + _require(uploaded[0]["chunk_text"] == "EXIT 12" + block, "The stored text must still carry the whole block.") + + # An image whose only text is its description must never fail ingestion for a long description. + _, embedded, uploaded = _run_save_chunks( + documents, "", VISION_ANALYSIS, max_characters=60, legacy_embedding=False, + ) + _require(embedded == [block.lstrip()[:60]], "A description-only chunk must embed its clamped description.") + _require(uploaded[0]["chunk_text"] == block.lstrip(), "The description-only chunk must store the whole block.") refused = False try: diff --git a/functional_tests/test_image_generation_api_version_default.py b/functional_tests/test_image_generation_api_version_default.py index 9389b31d0..19b352772 100644 --- a/functional_tests/test_image_generation_api_version_default.py +++ b/functional_tests/test_image_generation_api_version_default.py @@ -317,10 +317,12 @@ def test_v2_legacy_images_fallback_and_admin_field_use_the_new_default(): assert f"DEFAULT_IMAGES_API_VERSION = '{NEW_DEFAULT}'" in route_source assert "LEGACY_DEFAULT_IMAGES_API_VERSION = DEFAULT_IMAGES_API_VERSION" in route_source old_default_assignments = [ - node for node in ast.parse(route_source).body + node.targets[0].id for node in ast.parse(route_source).body if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant) and node.value.value == PREVIOUS_DEFAULT ] - assert not old_default_assignments, "No module constant may keep the old default" + assert old_default_assignments == ["LEGACY_APIM_DEFAULT_IMAGES_API_VERSION"], ( + f"Only the APIM fallback may keep the old default: {old_default_assignments}" + ) fields_tree = ast.parse((APP_ROOT / "admin_settings_fields.py").read_text(encoding="utf-8")) field_defaults = [ @@ -341,6 +343,29 @@ def test_v2_legacy_images_fallback_and_admin_field_use_the_new_default(): assert defaults.get("default") == NEW_DEFAULT, defaults +def test_legacy_images_fallback_changes_only_for_the_direct_route(): + """V2: a blank legacy direct version uses the new default; a blank APIM version does not change.""" + if str(APP_ROOT) not in sys.path: + sys.path.insert(0, str(APP_ROOT)) + from functions_image_api_route import resolve_image_generation_api_version # noqa: PLC0415 - pure module + + direct = { + "enable_image_generation": True, + "enable_image_gen_apim": False, + "azure_openai_image_gen_api_version": "", + "azure_apim_image_gen_api_version": "", + "image_gen_model": {"selected": [{"deploymentName": "legacy-image", "modelName": "gpt-image-1"}]}, + } + assert resolve_image_generation_api_version(direct) == NEW_DEFAULT + + apim = {**direct, "enable_image_gen_apim": True, "azure_apim_image_gen_deployment": "gateway-image"} + assert resolve_image_generation_api_version(apim) == PREVIOUS_DEFAULT, ( + "A blank APIM version must keep the fallback the gateway has always received" + ) + assert resolve_image_generation_api_version({**apim, "azure_apim_image_gen_api_version": "2024-02-01"}) == "2024-02-01" + assert resolve_image_generation_api_version({**direct, "azure_openai_image_gen_api_version": "2025-03-01-preview"}) == "2025-03-01-preview" + + def _run_offline_probe(): # Real application imports must occur inside the external-I/O bootstrap seam. from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 diff --git a/ui_tests/test_v2_chat_image_uploads.py b/ui_tests/test_v2_chat_image_uploads.py index 3b835a85d..2b30de68e 100644 --- a/ui_tests/test_v2_chat_image_uploads.py +++ b/ui_tests/test_v2_chat_image_uploads.py @@ -515,3 +515,23 @@ def test_image_citation_viewer_shows_what_was_extracted(image_ui): expect(viewer.locator("img[alt='lab.png']")).to_be_visible() expect(viewer.get_by_text("What was extracted from this image")).to_be_visible() expect(viewer).to_contain_text(VISION_DESCRIPTION) + + +def test_uploaded_image_sources_explain_a_failed_ingestion(image_ui): + page, api = image_ui + failed = json.loads(json.dumps(WORKSPACE_EXTRACTION)) + failed["indexed_text_available"] = False + failed["workspace_document"].update({ + "status": "Error: Processing failed: The chunk is too large for the selected embedding model.", + "percentage_complete": 0, + "indexed_chunk_count": 0, + }) + api.file_content["sized-image"] = (200, failed) + mount_messages(page, api, [file_message("sized-image", "lab.png", "doc-lab")]) + + page.get_by_role("button", name="View the full-size image: lab.png").hover() + page.get_by_role("button", name="Show sources", exact=True).click() + + expect(page.get_by_role("region", name="AI vision analysis")).to_contain_text(VISION_DESCRIPTION) + expect(page.get_by_text("Processing this file failed, so nothing from it was indexed for search.", exact=False)).to_be_visible() + expect(page.get_by_text("Still processing", exact=False)).to_have_count(0) From d8bf2b674eac2578d99532ea5708148908ed8aa4 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 30 Sep 2026 14:24:51 -0400 Subject: [PATCH 6/6] Address CodeQL findings on the image description indexing tests Folded from Development: assert batch token usage in the no-vision case and add a fresh-process probe that imports functions_content_understanding before functions_content. The fix doc records the existing import cycle. Tests and docs only. (cherry picked from commit 60365d532) Refs #1583 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...SCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md | 12 +++++ ...e_description_indexing_without_ocr_text.py | 54 ++++++++++++++++--- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md index 04127d82f..6f8a03b1b 100644 --- a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md +++ b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md @@ -158,6 +158,10 @@ blocked. It covers: de-duplication, - the extraction fallback for images, analyzer failure and empty results, PDFs, document analyzer content and failure, and Standard extraction, plus the chat upload call site, +- the image analyzer fallback when `functions_content_understanding` is imported before + `functions_content`, in a separate fresh process. The two modules already reach each other + through `functions_settings`, `functions_document_actions`, and `functions_search`, so both + import orders are checked, - the vision connection lookup: match precedence, disabled endpoints and models, blank and malformed input, legacy fallback with multi-endpoint off or no match, and APIM, - unchanged metadata extraction client construction and every error message, @@ -187,6 +191,14 @@ suites. metadata extraction, so the image still has no generated title. - A vision model name that exists on more than one enabled connection resolves by the precedence above, because the setting stores only the name. +- `extract_content_with_extraction_engine()` still imports `functions_content_understanding` when + it runs, and that import is part of a pre-existing cycle: + `functions_content` → `functions_content_understanding` → `functions_settings` → + `functions_document_actions` → `functions_search` → `functions_content`. This fix adds + `analyze_image_with_content_understanding` to that existing import statement but no new module + dependency, so CodeQL reports the same `py/cyclic-import` finding at the statement's new + location. Breaking the cycle means changing the settings bootstrap chain and is left for separate + work. ## Validation diff --git a/functional_tests/test_image_description_indexing_without_ocr_text.py b/functional_tests/test_image_description_indexing_without_ocr_text.py index 40f5c7e01..b760ea26f 100644 --- a/functional_tests/test_image_description_indexing_without_ocr_text.py +++ b/functional_tests/test_image_description_indexing_without_ocr_text.py @@ -28,7 +28,9 @@ The model-name lookup and the chat upload call-site check run in-process. Checks that need the real application modules run in fresh normal and optimized processes with external I/O blocked, -stubbing only the Azure-facing calls. +stubbing only the Azure-facing calls. A separate fresh process imports functions_content_understanding +before functions_content, the reverse of the main probe, because the two modules reach each other +through a pre-existing import cycle, and checks that the image analyzer fallback still resolves. """ import ast @@ -603,6 +605,7 @@ def run(metadata, chunks): stored = [document["chunk_text"] for document in uploaded_documents] _require(stored == ["Page one"] and batched_inputs == [["Page one"]], "Without a vision analysis, empty batch chunks are skipped and others are unchanged.") + _require(usage["total_tokens"] == 5, "Skipped chunks must not add embedding token usage.") usage = run({"version": 1}, [{"page_text_content": " ", "page_number": 1, "file_name": "x.md"}]) _require(not batched_inputs and not uploaded_documents and usage["total_tokens"] == 0, @@ -1164,6 +1167,7 @@ def _run_offline_probe(): from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 with offline_app_imports() as offline, tempfile.TemporaryDirectory() as work_dir: + _require("functions_content" not in sys.modules, "functions_content was imported before the probe.") import functions_content as content # noqa: PLC0415 import functions_content_understanding as content_understanding # noqa: PLC0415 import functions_documents as documents # noqa: PLC0415 @@ -1186,25 +1190,63 @@ def _run_offline_probe(): print("All offline image description indexing checks passed.") -@pytest.mark.parametrize("optimized", (False, True)) -def test_image_description_indexing_real_modules(optimized): - """Run the real-module checks in a fresh process with external I/O blocked.""" +def _run_reverse_import_order_probe(): + """Import Content Understanding before functions_content, the reverse of the main probe. + + extract_content_with_extraction_engine() imports functions_content_understanding when it + runs, and functions_content_understanding reaches functions_content back through + functions_settings, functions_document_actions, and functions_search. The image analyzer + fallback must resolve both deferred names whichever module is imported first. + """ + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline, tempfile.TemporaryDirectory() as work_dir: + for module_name in ("functions_content", "functions_content_understanding"): + _require(module_name not in sys.modules, f"{module_name} was imported before the probe.") + import functions_content_understanding as content_understanding # noqa: PLC0415 + _require("functions_content" in sys.modules, + "Importing functions_content_understanding no longer reaches functions_content; update this probe.") + import functions_content as content # noqa: PLC0415 + _require(not offline.network_attempts, "Application imports attempted network access.") + + check_extraction_engine_image_fallback(content, content_understanding, work_dir) + _require(not offline.network_attempts, "The checks attempted network access.") + + print("Reverse import order checks passed.") + + +def _run_probe_process(probe_flag, optimized=False): + """Run a probe in a fresh interpreter so imports start from a clean module table.""" env = dict(os.environ) env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(APP_ROOT), str(TEST_ROOT))) env["PYTHONIOENCODING"] = "utf-8" command = [sys.executable] if optimized: command.append("-O") - command.extend((str(Path(__file__).resolve()), "--offline-probe")) - result = subprocess.run( + command.extend((str(Path(__file__).resolve()), probe_flag)) + return subprocess.run( command, cwd=ROOT, env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=300, ) + + +@pytest.mark.parametrize("optimized", (False, True)) +def test_image_description_indexing_real_modules(optimized): + """Run the real-module checks in a fresh process with external I/O blocked.""" + result = _run_probe_process("--offline-probe", optimized=optimized) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_image_analyzer_fallback_with_reverse_import_order(): + """The image analyzer fallback must work when Content Understanding is imported first.""" + result = _run_probe_process("--reverse-import-order-probe") assert result.returncode == 0, result.stdout + result.stderr if __name__ == "__main__": if "--offline-probe" in sys.argv: _run_offline_probe() + elif "--reverse-import-order-probe" in sys.argv: + _run_reverse_import_order_probe() else: raise SystemExit(pytest.main([__file__, "-q"]))