diff --git a/application/external_apps/databaseseeder/artifacts/admin_settings.json b/application/external_apps/databaseseeder/artifacts/admin_settings.json index 2bbd3c5a1..e7408708d 100644 --- a/application/external_apps/databaseseeder/artifacts/admin_settings.json +++ b/application/external_apps/databaseseeder/artifacts/admin_settings.json @@ -37,7 +37,7 @@ "enable_image_generation": false, "enable_image_gen_apim": false, "azure_openai_image_gen_endpoint": "", - "azure_openai_image_gen_api_version": "2024-05-01-preview", + "azure_openai_image_gen_api_version": "2025-04-01-preview", "azure_openai_image_gen_authentication_type": "key", "azure_openai_image_gen_subscription_id": "", "azure_openai_image_gen_resource_group": "", diff --git a/application/single_app/config.py b/application/single_app/config.py index 79c93e1a6..f4185657d 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -98,7 +98,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.046" +VERSION = "0.261.047" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index 9f57cb90d..440f68d90 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -621,12 +621,41 @@ def resolve_extraction_engine_for_mode(extraction_mode, settings): return functions_settings.resolve_enhanced_extraction_engine(settings) +# Persisted as the extraction engine reason and shown in workspace tooltips, so keep it short. +CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON = ( + 'Content Understanding found no text, so its image analyzer description was indexed' +) + + +def _describe_image_with_content_understanding(file_path, settings, analyze_image): + """Return the Content Understanding image analyzer description, or '' when it has none.""" + try: + description_text = str(analyze_image(file_path, settings=settings) or '').strip() + except Exception as image_analyzer_error: + log_event( + f"[EXTRACTION_ENGINE] Content Understanding image analyzer failed for " + f"{os.path.basename(file_path)}: {image_analyzer_error}. " + "Falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + return '' + + if not description_text: + log_event( + f"[EXTRACTION_ENGINE] Content Understanding image analyzer returned no description for " + f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.", + level=logging.WARNING, + ) + return description_text + + def extract_content_with_extraction_engine( file_path, extraction_mode, extraction_engine=None, settings=None, pages=None, + is_image=False, ): """Extract page content with the engine that backs the requested extraction mode. @@ -634,6 +663,11 @@ def extract_content_with_extraction_engine( prefers Azure AI Content Understanding, but always falls back to Document Intelligence ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion. + When ``is_image`` is set and the document analyzer finds no content, which is normal for a + photo with no text, the Content Understanding image analyzer's description is returned as + the image's single page instead. Only if that also yields nothing does extraction fall back + to Document Intelligence Layout. + Returns ``(pages_data, engine_used, fallback_reason)``. """ resolved_settings = settings if settings is not None else get_settings() @@ -650,7 +684,10 @@ def extract_content_with_extraction_engine( '', ) - from functions_content_understanding import extract_content_with_content_understanding + from functions_content_understanding import ( + analyze_image_with_content_understanding, + extract_content_with_content_understanding, + ) try: content_understanding_pages = extract_content_with_content_understanding( @@ -664,6 +701,18 @@ def extract_content_with_extraction_engine( functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, '', ) + if is_image: + image_description = _describe_image_with_content_understanding( + file_path, + resolved_settings, + analyze_image_with_content_understanding, + ) + if image_description: + return ( + [{'page_number': 1, 'content': image_description}], + functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING, + CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON, + ) fallback_reason = ( 'Content Understanding returned no content, so Document Intelligence Layout was used' ) diff --git a/application/single_app/functions_content_understanding.py b/application/single_app/functions_content_understanding.py index c78098852..22bd8b7c2 100644 --- a/application/single_app/functions_content_understanding.py +++ b/application/single_app/functions_content_understanding.py @@ -14,6 +14,7 @@ import json import logging import os +import re import time import requests @@ -48,6 +49,11 @@ "Understanding defaults before using Enhanced extraction." ) +# Image analyzers such as prebuilt-imageSearch return a bare image reference, for example +# "![image](pages/1)", as the markdown and put the actual description in a Summary field. +MARKDOWN_IMAGE_PLACEHOLDER_PATTERN = re.compile(r'!\[[^\]]*\]\([^)]*\)') +CONTENT_UNDERSTANDING_SUMMARY_FIELD = 'summary' + class ContentUnderstandingError(Exception): """Raised when a Content Understanding request cannot be completed.""" @@ -521,8 +527,45 @@ def extract_content_with_content_understanding(file_path, pages=None, settings=N return pages_data +def _is_image_placeholder_markdown(markdown_text): + """Return True when markdown holds nothing but image references such as ``![image](pages/1)``.""" + return not MARKDOWN_IMAGE_PLACEHOLDER_PATTERN.sub('', str(markdown_text or '')).strip() + + +def _build_string_field_blocks(content): + """Return readable text for the non-empty string fields an analyzer extracted. + + The ``Summary`` field is emitted as plain text first, because it is the analyzer's own + description of the input. Every other string field is emitted as ``Name: value``. + """ + fields = content.get('fields') + if not isinstance(fields, dict): + return [] + + summary_blocks = [] + field_blocks = [] + for field_name, field_value in fields.items(): + if not isinstance(field_value, dict): + continue + value_string = field_value.get('valueString') + if not isinstance(value_string, str) or not value_string.strip(): + continue + + if str(field_name).strip().lower() == CONTENT_UNDERSTANDING_SUMMARY_FIELD: + summary_blocks.append(value_string.strip()) + else: + field_blocks.append(f"{field_name}: {value_string.strip()}") + + return summary_blocks + field_blocks + + def analyze_image_with_content_understanding(image_path, settings=None): - """Analyze a standalone image and return its description text, or an empty string.""" + """Analyze a standalone image and return its description text, or an empty string. + + Image analyzers describe the image in extracted fields rather than in the markdown, which is + only a placeholder image reference, so string fields are read alongside any real markdown + and figure descriptions. + """ resolved_settings = settings if settings is not None else _get_settings() config = _resolve_config(settings=resolved_settings) @@ -535,15 +578,27 @@ def analyze_image_with_content_understanding(image_path, settings=None): text_blocks = [] for content in _iter_document_contents(result): markdown_text = str(content.get('markdown') or '').strip() + if _is_image_placeholder_markdown(markdown_text): + markdown_text = '' if markdown_text: text_blocks.append(markdown_text) + field_blocks = _build_string_field_blocks(content) + text_blocks.extend(field_blocks) + + # De-duplicate figure descriptions against everything already emitted for this content. + emitted_text = "\n\n".join([markdown_text] + field_blocks) for summary in _build_figure_summaries(content): - rendered_summary = _render_figure_summary(summary, markdown_text) + rendered_summary = _render_figure_summary(summary, emitted_text) if rendered_summary: text_blocks.append(rendered_summary) - return "\n\n".join(block for block in text_blocks if block).strip() + description_text = "\n\n".join(block for block in text_blocks if block).strip() + debug_print( + f"[CONTENT_UNDERSTANDING] Image analyzer {config['image_analyzer_id']} returned " + f"{len(description_text)} character(s) for {os.path.basename(image_path)}" + ) + return description_text def test_content_understanding_connection(config_override, sample_file_path=None): diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index c3c49d0c8..d3a879016 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -58,6 +58,7 @@ validate_xml_bytes, ) from functions_model_endpoint_types import ( + find_enabled_model_endpoint_for_model_name, get_model_endpoint_api_type, resolve_model_endpoint_request_model, ) @@ -200,6 +201,93 @@ def _build_model_endpoint_client( return client +def _build_model_endpoint_client_for_model( + settings, + endpoint_cfg, + model_id, + *, + fallback_endpoint_id='', + fallback_provider='', + purpose_label='model', + identity_context=None, +): + """Build a sync chat client for one model on a configured AI connection. + + Resolves the connection's Key Vault secrets first, then finds the model by id and validates + the provider and connection details. ``purpose_label`` names the feature in error messages, + which never include secrets. + + Returns ``(client, request_model)``. + """ + endpoint_cfg = keyvault_model_endpoint_get_helper( + endpoint_cfg, + endpoint_cfg.get("id") or fallback_endpoint_id, + scope="global", + return_type=SecretReturnType.VALUE, + ) + + models = endpoint_cfg.get("models", []) or [] + model_cfg = next((m for m in models if m.get("id") == model_id), None) + if not model_cfg: + raise LookupError(f"Selected {purpose_label} model could not be found on the endpoint.") + if not model_cfg.get("enabled", True): + raise ValueError(f"Selected {purpose_label} model is disabled.") + + provider = str(endpoint_cfg.get("provider") or fallback_provider or "aoai").lower() + connection = endpoint_cfg.get("connection", {}) or {} + auth_settings = endpoint_cfg.get("auth", {}) or {} + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + endpoint = str(connection.get("endpoint") or "").strip() + api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() + api_type = get_model_endpoint_api_type(endpoint_cfg) + anthropic_version = str(connection.get("anthropic_version") or "").strip() + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) + + if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: + raise ValueError(f"Selected {purpose_label} provider '{provider}' is not supported.") + if not endpoint or not deployment or ( + runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version + ): + raise ValueError(f"Selected {purpose_label} endpoint is incomplete.") + + return _build_model_endpoint_client( + auth_settings, + provider, + endpoint, + api_version, + deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get("allow_private_custom_model_endpoints", False) + ), + settings=settings, + endpoint_config=endpoint_cfg, + identity_context=identity_context, + ), deployment + + +def resolve_vision_model_endpoint(settings, vision_model): + """Return the enabled AI connection and model that serve the configured vision model. + + The vision model setting stores only a model name, so it is matched against enabled + endpoints and models when multi-endpoint models are enabled. Returns ``(endpoint, model)`` + from the normalized endpoint list, or ``(None, None)`` when the legacy GPT connection + should be used instead. + """ + settings = settings or {} + if not settings.get("enable_multi_model_endpoints", False): + return None, None + + endpoints, _ = normalize_model_endpoints(settings.get("model_endpoints", []) or []) + return find_enabled_model_endpoint_for_model_name(endpoints, vision_model) + + def _resolve_metadata_extraction_client(settings, identity_context=None): selection = _normalize_model_endpoint_selection(settings.get("metadata_extraction_model_selection")) @@ -215,57 +303,15 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): if not endpoint_cfg.get("enabled", True): raise ValueError("Selected metadata extraction endpoint is disabled.") - endpoint_cfg = keyvault_model_endpoint_get_helper( + return _build_model_endpoint_client_for_model( + settings, endpoint_cfg, - endpoint_cfg.get("id") or selection["endpoint_id"], - scope="global", - return_type=SecretReturnType.VALUE, - ) - - models = endpoint_cfg.get("models", []) or [] - model_cfg = next((m for m in models if m.get("id") == selection["model_id"]), None) - if not model_cfg: - raise LookupError("Selected metadata extraction model could not be found on the endpoint.") - if not model_cfg.get("enabled", True): - raise ValueError("Selected metadata extraction model is disabled.") - - provider = str(endpoint_cfg.get("provider") or selection["provider"] or "aoai").lower() - connection = endpoint_cfg.get("connection", {}) or {} - auth_settings = endpoint_cfg.get("auth", {}) or {} - deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) - endpoint = str(connection.get("endpoint") or "").strip() - api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() - api_type = get_model_endpoint_api_type(endpoint_cfg) - anthropic_version = str(connection.get("anthropic_version") or "").strip() - runtime_protocol = infer_model_endpoint_protocol( - provider, - endpoint, - deployment, - api_type, - ) - - if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: - raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.") - if not endpoint or not deployment or ( - runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version - ): - raise ValueError("Selected metadata extraction endpoint is incomplete.") - - return _build_model_endpoint_client( - auth_settings, - provider, - endpoint, - api_version, - deployment, - api_type=api_type, - anthropic_version=anthropic_version, - allow_private_custom_endpoints=bool( - settings.get("allow_private_custom_model_endpoints", False) - ), - settings=settings, - endpoint_config=endpoint_cfg, + selection["model_id"], + fallback_endpoint_id=selection["endpoint_id"], + fallback_provider=selection["provider"], + purpose_label="metadata extraction", identity_context=identity_context, - ), deployment + ) gpt_model = settings.get('metadata_extraction_model') if not gpt_model: @@ -320,6 +366,18 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): # treats them as a signal that Enhanced extraction is worth the extra cost, because Content # Understanding is the only engine that describes figures. DI_MARKDOWN_FIGURE_PATTERN = re.compile(r'(|!\[[^\]]*\]\()', re.IGNORECASE) +# Document Intelligence Layout emits page numbers and page breaks as markdown comments, for example +# . They annotate the page rather than carry its text, so an image whose OCR +# output is only these annotations has no text worth indexing on its own. Header and footer comments +# are deliberately not matched, because their quoted values are the recognized text itself. +DI_MARKDOWN_PAGE_ANNOTATION_PATTERN = re.compile( + r'', + re.IGNORECASE, +) +VISION_ANALYSIS_CONTENT_FIELDS = ('description', 'objects', 'text', 'analysis') +# Marks the single chunk saved for an image that has no OCR text but does have an AI vision +# description, so the save loop keeps it and save_chunks indexes the description on its own. +VISION_DESCRIPTION_ONLY_CHUNK_FLAG = 'vision_description_only' # Budget for image content merged into an existing chunk. The limit is the embedding character # budget, so a merged chunk can never grow past what the embedding endpoint accepts. OFFICE_IMAGE_MERGE_MIN_CHAR_LIMIT = 4000 @@ -699,6 +757,7 @@ def _extract_pages_with_extraction_engine( extraction_engine, settings=None, pages=None, + is_image=False, ): """Extract page content with the resolved engine, falling back to Document Intelligence Layout.""" return extract_content_with_extraction_engine( @@ -707,6 +766,7 @@ def _extract_pages_with_extraction_engine( extraction_engine=extraction_engine, settings=settings, pages=pages, + is_image=is_image, ) @@ -3487,12 +3547,77 @@ def update_document(**kwargs): # print(f"Failed to update status to error state for {document_id}: {inner_e}") raise # Re-raise the original exception +def _is_usable_vision_analysis(vision_analysis): + """Return True when a vision analysis succeeded and describes something worth indexing.""" + if not isinstance(vision_analysis, dict) or 'error' in vision_analysis: + return False + + for field_name in VISION_ANALYSIS_CONTENT_FIELDS: + field_value = vision_analysis.get(field_name) + if isinstance(field_value, str): + if field_value.strip(): + return True + elif field_value: + return True + return False + + +def _format_vision_analysis_block(vision_analysis): + """Format a vision analysis as the searchable block appended to an image's chunk text. + + Returns an empty string for a missing, failed, or empty analysis, so an error message from + the vision model is never indexed as if it described the image. + """ + if not _is_usable_vision_analysis(vision_analysis): + return '' + + vision_text_parts = ["\n\n=== AI Vision Analysis ==="] + vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") + + if vision_analysis.get('description'): + vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") + + if vision_analysis.get('objects'): + objects_list = vision_analysis['objects'] + if isinstance(objects_list, list): + vision_text_parts.append(f"\nObjects Detected: {', '.join(str(item) for item in objects_list)}") + else: + vision_text_parts.append(f"\nObjects Detected: {objects_list}") + + if vision_analysis.get('text'): + vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") + + if vision_analysis.get('analysis'): + vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") + + return "\n".join(vision_text_parts) + + +def _append_vision_block_to_chunk_text(chunk_text, vision_text): + """Append a vision block to chunk text, dropping its leading blank lines when there is no text.""" + chunk_text = chunk_text or '' + if not vision_text: + return chunk_text + if not chunk_text.strip(): + return vision_text.lstrip() + return chunk_text + vision_text + + +def _has_indexable_ocr_text(page_content): + """Return True when OCR output has text beyond Document Intelligence page-number annotations.""" + text_without_annotations = DI_MARKDOWN_PAGE_ANNOTATION_PATTERN.sub('', str(page_content or '')) + return bool(text_without_annotations.strip()) + + def save_chunks(page_text_content, page_number, file_name, user_id, document_id, group_id=None, public_workspace_id=None): """ Save a single chunk (one page) at a time: - - Generate embedding + - Build the chunk text, appending the document's AI vision analysis when it has one + - Generate the embedding from that same text - Build chunk metadata - Upload to Search index + + A chunk with no text to index, and no vision analysis to stand in for it, is not written. """ current_time = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') is_group = group_id is not None @@ -3551,11 +3676,37 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, print(f"Error updating document status or retrieving metadata for document {document_id}: {repr(e)}\nTraceback:\n{traceback.format_exc()}") raise + # Build the stored chunk text before embedding it, so an image's vector reflects its AI vision + # description rather than only whatever OCR text it happened to contain. + enhanced_chunk_text = page_text_content or '' + vision_text = _format_vision_analysis_block(metadata.get('vision_analysis')) + if vision_text: + enhanced_chunk_text = _append_vision_block_to_chunk_text(enhanced_chunk_text, vision_text) + debug_print( + f"[SAVE_CHUNKS] Appended vision analysis for document {document_id}: " + f"chunk_text length {len(enhanced_chunk_text)} (original: {len(page_text_content or '')}, " + f"vision: {len(vision_text)})" + ) + else: + debug_print(f"[SAVE_CHUNKS] No usable vision analysis found for document {document_id}") + + if not enhanced_chunk_text.strip(): + log_event( + "[SAVE_CHUNKS] Skipped a chunk with no text or vision description to index.", + extra={ + "document_id": document_id, + "page_number": page_number, + "file_name": file_name, + }, + level=logging.WARNING + ) + return None + # Generate embedding try: #status = f"Generating embedding for page {page_number}" #update_document(document_id=document_id, user_id=user_id, status=status) - embedding_input = page_text_content + embedding_input = enhanced_chunk_text max_embedding_characters = get_embedding_safe_chunk_characters() # Last-resort guard. Every processor bounds its own chunks, so reaching this means content @@ -3591,40 +3742,6 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, title = metadata.get('title', '') if metadata else '' document_classification = metadata.get('document_classification', 'None') if metadata else 'None' - # Check if this document has vision analysis and append it to chunk_text - vision_analysis = metadata.get('vision_analysis') - enhanced_chunk_text = page_text_content - - if vision_analysis: - debug_print(f"[SAVE_CHUNKS] Document {document_id} has vision analysis, appending to chunk_text") - # Format vision analysis as structured text for better searchability - vision_text_parts = [] - vision_text_parts.append("\n\n=== AI Vision Analysis ===") - vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") - - if vision_analysis.get('description'): - vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") - - if vision_analysis.get('objects'): - objects_list = vision_analysis['objects'] - if isinstance(objects_list, list): - vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") - else: - vision_text_parts.append(f"\nObjects Detected: {objects_list}") - - if vision_analysis.get('text'): - vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") - - if vision_analysis.get('analysis'): - vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") - - vision_text = "\n".join(vision_text_parts) - enhanced_chunk_text = page_text_content + vision_text - - debug_print(f"[SAVE_CHUNKS] Enhanced chunk_text length: {len(enhanced_chunk_text)} (original: {len(page_text_content)}, vision: {len(vision_text)})") - else: - debug_print(f"[SAVE_CHUNKS] No vision analysis found for document {document_id}") - if is_public_workspace: chunk_document = { "id": chunk_id, @@ -3770,44 +3887,43 @@ def save_chunks_batch(chunks_data, user_id, document_id, group_id=None, public_w log_event(f"[SAVE_CHUNKS_BATCH] Error retrieving metadata for document {document_id}: {repr(e)}", level=logging.ERROR) raise + # Build every chunk's stored text first and embed that same text, matching save_chunks(). + vision_text = _format_vision_analysis_block(metadata.get('vision_analysis')) + chunks_to_save = [] + for chunk_info in chunks_data: + enhanced_chunk_text = _append_vision_block_to_chunk_text(chunk_info['page_text_content'], vision_text) + if not enhanced_chunk_text.strip(): + log_event( + "[SAVE_CHUNKS_BATCH] Skipped a chunk with no text or vision description to index.", + extra={ + "document_id": document_id, + "page_number": chunk_info.get('page_number'), + "file_name": chunk_info.get('file_name'), + }, + level=logging.WARNING + ) + continue + chunks_to_save.append((chunk_info, enhanced_chunk_text)) + + total_token_usage = {'total_tokens': 0, 'prompt_tokens': 0, 'model_deployment_name': None} + if not chunks_to_save: + return total_token_usage + # Generate all embeddings in batches - texts = [c['page_text_content'] for c in chunks_data] + texts = [enhanced_chunk_text for _, enhanced_chunk_text in chunks_to_save] try: embedding_results = generate_embeddings_batch(texts) except Exception as e: log_event(f"[SAVE_CHUNKS_BATCH] Error generating batch embeddings for document {document_id}: {e}", level=logging.ERROR) raise - # Check for vision analysis once - vision_analysis = metadata.get('vision_analysis') - vision_text = "" - if vision_analysis: - vision_text_parts = [] - vision_text_parts.append("\n\n=== AI Vision Analysis ===") - vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") - if vision_analysis.get('description'): - vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") - if vision_analysis.get('objects'): - objects_list = vision_analysis['objects'] - if isinstance(objects_list, list): - vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") - else: - vision_text_parts.append(f"\nObjects Detected: {objects_list}") - if vision_analysis.get('text'): - vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") - if vision_analysis.get('analysis'): - vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") - vision_text = "\n".join(vision_text_parts) - # Build all chunk documents chunk_documents = [] - total_token_usage = {'total_tokens': 0, 'prompt_tokens': 0, 'model_deployment_name': None} - for idx, chunk_info in enumerate(chunks_data): + for idx, (chunk_info, enhanced_chunk_text) in enumerate(chunks_to_save): embedding, token_usage = embedding_results[idx] page_number = chunk_info['page_number'] file_name = chunk_info['file_name'] - page_text_content = chunk_info['page_text_content'] if token_usage: total_token_usage['total_tokens'] += token_usage.get('total_tokens', 0) @@ -3816,7 +3932,6 @@ def save_chunks_batch(chunks_data, user_id, document_id, group_id=None, public_w total_token_usage['model_deployment_name'] = token_usage.get('model_deployment_name') chunk_id = f"{document_id}_{page_number}" - enhanced_chunk_text = page_text_content + vision_text if vision_text else page_text_content if is_public_workspace: chunk_document = { @@ -5815,49 +5930,85 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): print(f"Warning: Multi-modal vision enabled but no model selected") return None - # Initialize client (reuse Chat Model) - enable_gpt_apim = settings.get('enable_gpt_apim', False) - debug_print(f"[VISION_ANALYSIS] Using APIM: {enable_gpt_apim}") - - if enable_gpt_apim: - api_version = settings.get('azure_apim_gpt_api_version') - endpoint = settings.get('azure_apim_gpt_endpoint') - debug_print(f"[VISION_ANALYSIS] APIM Configuration:") - debug_print(f" Endpoint: {endpoint}") - debug_print(f" API Version: {api_version}") - - gpt_client = AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=settings.get('azure_apim_gpt_subscription_key') + # The setting stores only a model name. With multi-endpoint models enabled, route the call + # through the AI connection that hosts that model; otherwise use the legacy GPT connection. + request_model = vision_model + vision_endpoint_cfg, vision_model_cfg = resolve_vision_model_endpoint(settings, vision_model) + if vision_endpoint_cfg and vision_model_cfg: + gpt_client, request_model = _build_model_endpoint_client_for_model( + settings, + vision_endpoint_cfg, + vision_model_cfg.get('id'), + fallback_endpoint_id=str(vision_endpoint_cfg.get('id') or ''), + purpose_label='vision', + identity_context={'user_id': user_id}, + ) + log_event( + "[VISION_ANALYSIS] Using the AI connection that hosts the vision model.", + extra={ + "document_id": document_id, + "vision_model": vision_model, + "request_model": request_model, + "endpoint_id": vision_endpoint_cfg.get('id'), + "endpoint_name": vision_endpoint_cfg.get('name'), + "model_id": vision_model_cfg.get('id'), + "provider": vision_endpoint_cfg.get('provider'), + }, ) else: - # Use managed identity or key - auth_type = settings.get('azure_openai_gpt_authentication_type', 'key') - api_version = settings.get('azure_openai_gpt_api_version') - endpoint = settings.get('azure_openai_gpt_endpoint') - - debug_print(f"[VISION_ANALYSIS] Direct Azure OpenAI Configuration:") - debug_print(f" Endpoint: {endpoint}") - debug_print(f" API Version: {api_version}") - debug_print(f" Auth Type: {auth_type}") - - if auth_type == 'managed_identity': - token_provider = get_bearer_token_provider( - DefaultAzureCredential(), - cognitive_services_scope - ) + # Initialize client (reuse Chat Model) + enable_gpt_apim = settings.get('enable_gpt_apim', False) + debug_print(f"[VISION_ANALYSIS] Using APIM: {enable_gpt_apim}") + + if enable_gpt_apim: + api_version = settings.get('azure_apim_gpt_api_version') + endpoint = settings.get('azure_apim_gpt_endpoint') + debug_print(f"[VISION_ANALYSIS] APIM Configuration:") + debug_print(f" Endpoint: {endpoint}") + debug_print(f" API Version: {api_version}") + gpt_client = AzureOpenAI( api_version=api_version, azure_endpoint=endpoint, - azure_ad_token_provider=token_provider + api_key=settings.get('azure_apim_gpt_subscription_key') ) else: - gpt_client = AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=settings.get('azure_openai_gpt_key') - ) + # Use managed identity or key + auth_type = settings.get('azure_openai_gpt_authentication_type', 'key') + api_version = settings.get('azure_openai_gpt_api_version') + endpoint = settings.get('azure_openai_gpt_endpoint') + + debug_print(f"[VISION_ANALYSIS] Direct Azure OpenAI Configuration:") + debug_print(f" Endpoint: {endpoint}") + debug_print(f" API Version: {api_version}") + debug_print(f" Auth Type: {auth_type}") + + if auth_type == 'managed_identity': + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + cognitive_services_scope + ) + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider + ) + else: + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=settings.get('azure_openai_gpt_key') + ) + + log_event( + "[VISION_ANALYSIS] Using the legacy GPT connection for the vision model.", + extra={ + "document_id": document_id, + "vision_model": vision_model, + "connection": "apim" if enable_gpt_apim else "azure_openai", + "multi_endpoint_enabled": bool(settings.get('enable_multi_model_endpoints', False)), + }, + ) # Create vision prompt print(f"Analyzing image with vision model: {vision_model}") @@ -5901,7 +6052,7 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): }""" api_params = { - "model": vision_model, + "model": request_model, "messages": [ { "role": "user", @@ -6045,6 +6196,15 @@ def analyze_image_with_vision_model(image_path, user_id, document_id, settings): except Exception as e: print(f"Error in vision analysis for {document_id}: {str(e)}") traceback.print_exc() + log_event( + "[VISION_ANALYSIS] Vision analysis failed, so no vision description was produced.", + extra={ + "document_id": document_id, + "vision_model": settings.get('multimodal_vision_model') if isinstance(settings, dict) else None, + "error_type": type(e).__name__, + }, + level=logging.WARNING, + ) return None def upload_to_blob(temp_file_path, user_id, document_id, blob_filename, update_callback, group_id=None, public_workspace_id=None, mark_enhanced_citations=True): @@ -9048,6 +9208,9 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, update_callback(num_file_chunks=num_file_chunks, status=f"Processing {original_filename} in {num_file_chunks} file chunk(s)") total_final_chunks_processed = 0 + # Only this run's vision analysis may stand in for missing OCR text. A reprocessed document can + # still carry an older analysis on its record, which must not be indexed on its own. + vision_analysis_usable = False for idx, chunk_path in enumerate(file_paths_to_process, start=1): chunk_base_name, chunk_ext_loop = os.path.splitext(original_filename) chunk_effective_filename = original_filename @@ -9112,6 +9275,7 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, extraction_mode=document_intelligence_extraction_mode, extraction_engine=extraction_engine, settings=settings, + is_image=is_image, ) if engine_fallback_reason: extraction_engine = engine_used @@ -9165,6 +9329,7 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, 'status': "AI vision analysis completed" } update_callback(**update_fields) + vision_analysis_usable = _is_usable_vision_analysis(vision_analysis) print(f"Vision analysis saved to document metadata and will be appended to chunk_text for AI Search indexing") else: print(f"Vision analysis returned no results for: {chunk_effective_filename}") @@ -9220,10 +9385,38 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, status=f"Grouped {len(final_chunks_to_save)} chunk(s) for {chunk_effective_filename} using {target_size} page(s)/slide(s) per chunk." ) elif is_image: - if di_extracted_pages: - if 'page_number' not in di_extracted_pages[0]: di_extracted_pages[0]['page_number'] = 1 - final_chunks_to_save = di_extracted_pages - else: final_chunks_to_save = [] # No text extracted + # Keep only pages with real OCR text. Layout can annotate a photo with nothing but a + # stray page-number comment, which would otherwise become the image's only chunk. + image_text_pages = [ + page for page in di_extracted_pages + if isinstance(page, dict) and _has_indexable_ocr_text(page.get('content')) + ] + if image_text_pages: + if 'page_number' not in image_text_pages[0]: + image_text_pages[0]['page_number'] = 1 + final_chunks_to_save = image_text_pages + elif vision_analysis_usable: + # No OCR text, but the vision model described the image, so index that description + # on its own. save_chunks() appends and embeds the stored vision analysis. + final_chunks_to_save = [{ + 'page_number': 1, + 'content': '', + VISION_DESCRIPTION_ONLY_CHUNK_FLAG: True, + }] + log_event( + "[VISION_ANALYSIS] Image had no OCR text, so its AI vision description was indexed on its own.", + extra={ + "document_id": document_id, + "file_name": chunk_effective_filename, + "ocr_page_count": len(di_extracted_pages), + }, + ) + update_callback( + number_of_pages=1, + status=f"No text found in image {chunk_effective_filename}; indexing its AI vision description." + ) + else: + final_chunks_to_save = [] # No text extracted and no vision description to index # --- Embedded Office image analysis (DOCX/DOC/PPTX/PPT) --- # Neither extraction engine describes figures inside Office files, so embedded images are @@ -9297,7 +9490,9 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, chunk_index = chunk_data.get("page_number", i + 1) # Ensure page number exists chunk_content = chunk_data.get("content", "") - if not chunk_content.strip(): + # A description-only image chunk has no OCR text by design; save_chunks() + # supplies its text from the stored vision analysis. + if not chunk_content.strip() and not chunk_data.get(VISION_DESCRIPTION_ONLY_CHUNK_FLAG): print(f"Skipping empty chunk index {chunk_index} for {chunk_effective_filename}.") continue diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py index 61b3c0646..7d8307b11 100644 --- a/application/single_app/functions_model_endpoint_types.py +++ b/application/single_app/functions_model_endpoint_types.py @@ -7,7 +7,7 @@ the registry so an API type is declared in exactly one place. """ -from typing import Any, Dict +from typing import Any, Dict, Optional, Tuple from functions_model_endpoint_providers import ( DEFAULT_ANTHROPIC_VERSION, @@ -30,6 +30,7 @@ "MODEL_ENDPOINT_API_TYPE_OPENAI", "MODEL_ENDPOINT_CUSTOM_API_TYPES", "MODEL_ENDPOINT_PROVIDER_CUSTOM", + "find_enabled_model_endpoint_for_model_name", "get_model_endpoint_api_type", "normalize_model_endpoint_api_type", "resolve_model_endpoint_request_model", @@ -77,3 +78,59 @@ def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: or model_data.get("name") or "" ).strip() + + +def _get_model_name_match_rank(endpoint: Dict[str, Any], model: Dict[str, Any], model_name: str) -> Optional[int]: + """Return how strongly a model matches a stored name, lower being stronger, or None.""" + deployment_name = str(model.get("deploymentName") or "").strip() + if model_name in (deployment_name, resolve_model_endpoint_request_model(endpoint, model)): + return 0 + if str(model.get("modelName") or "").strip() == model_name: + return 1 + if str(model.get("id") or "").strip() == model_name: + return 2 + return None + + +def find_enabled_model_endpoint_for_model_name( + endpoints: Any, + model_name: Any, +) -> Tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]]]: + """Return the enabled endpoint and enabled model that serve a stored model name. + + Some settings, such as the multi-modal vision model, store only a model name rather than + an endpoint and model id. This resolves that name against the given endpoint records + without reading settings or secrets, so callers pass already-normalized endpoints. + + A deployment name or the resolved request model wins over a friendly model name, which + wins over a model id. Within the same precedence, the first match in endpoint order wins, + so a name that is a deployment on one connection and only a model family name on another + resolves to the connection that actually deploys it. Records without an ``enabled`` flag + count as enabled, matching ``normalize_model_endpoints``. + + Returns ``(endpoint, model)``, or ``(None, None)`` when nothing matches. + """ + requested_name = str(model_name or "").strip() + if not requested_name or not isinstance(endpoints, (list, tuple)): + return None, None + + best_match = (None, None) + best_rank = None + for endpoint in endpoints: + if not isinstance(endpoint, dict) or not endpoint.get("enabled", True): + continue + models = endpoint.get("models") + if not isinstance(models, (list, tuple)): + continue + for model in models: + if not isinstance(model, dict) or not model.get("enabled", True): + continue + rank = _get_model_name_match_rank(endpoint, model, requested_name) + if rank is None or (best_rank is not None and rank >= best_rank): + continue + best_match = (endpoint, model) + best_rank = rank + if rank == 0: + return best_match + + return best_match diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 361cefc20..58eabf9b9 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -1315,6 +1315,46 @@ def normalize_tabular_parity_durable_preflight_defaults(settings): return changed +# 2025-04-01-preview is the newest dated Azure OpenAI API version, and the one Microsoft documents +# for gpt-image-1, gpt-image-1.5, and gpt-image-2. +AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT = '2025-04-01-preview' +# The previous shipped default. It predates gpt-image model support. +AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT = '2024-12-01-preview' + + +def normalize_image_generation_api_version_settings(settings): + """Upgrade a stored Azure OpenAI image generation API version the admin never customized. + + deep_merge_dicts() only fills in keys that are *missing* from a persisted settings document, + so every deployment that loaded settings before the default changed still stores the old + 2024-12-01-preview default, which predates gpt-image model support. Raising the code-level + default alone would therefore never reach those deployments. + + A stored azure_openai_image_gen_api_version that is exactly the previous default, or blank, + was never chosen by an admin, so it is upgraded to AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT. + Any other value is a deliberate admin choice and is left untouched. + + azure_apim_image_gen_api_version is intentionally not migrated. An APIM gateway defines the + API version it accepts, which is why that setting has no default at all; rewriting it could + break a working gateway. + + Returns True when the settings were changed. + """ + if not isinstance(settings, dict): + return False + + stored_version = settings.get('azure_openai_image_gen_api_version') + if stored_version is not None and not isinstance(stored_version, str): + return False + + normalized_version = (stored_version or '').strip() + if normalized_version and normalized_version != AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT: + return False + + settings['azure_openai_image_gen_api_version'] = AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT + return True + + def get_settings(use_cosmos=False, include_source=False): default_settings = { # External health check @@ -1542,7 +1582,7 @@ def get_settings(use_cosmos=False, include_source=False): 'enable_image_generation': False, 'enable_image_gen_apim': False, 'azure_openai_image_gen_endpoint': '', - 'azure_openai_image_gen_api_version': '2024-12-01-preview', + 'azure_openai_image_gen_api_version': AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT, 'azure_openai_image_gen_authentication_type': 'key', 'azure_openai_image_gen_subscription_id': '', 'azure_openai_image_gen_resource_group': '', @@ -2005,6 +2045,7 @@ def normalize_loaded_settings(settings_item): normalize_key_vault_reminder_settings(merged) normalize_model_endpoint_identity_header_settings(merged) normalize_tabular_parity_durable_preflight_defaults(merged) + normalize_image_generation_api_version_settings(merged) merged['enable_tabular_processing_plugin'] = is_tabular_processing_enabled(merged) diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 49c54719d..8598c67ee 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -1412,6 +1412,16 @@ def _test_multimodal_vision_connection(payload): try: multi_endpoint_selection = payload.get('multi_endpoint') if isinstance(payload.get('multi_endpoint'), dict) else None + # Ingestion resolves the saved vision model name to the AI connection that hosts it, so the + # test uses that same connection whenever the lookup succeeds, even if APIM is selected. + runtime_endpoint, runtime_model = resolve_vision_model_endpoint(get_settings(), vision_model) + if runtime_endpoint and runtime_model: + multi_endpoint_selection = { + 'endpoint_id': str(runtime_endpoint.get('id') or '').strip(), + 'model_id': str(runtime_model.get('id') or '').strip(), + 'provider': str(runtime_endpoint.get('provider') or '').strip(), + 'deployment_name': resolve_model_endpoint_request_model(runtime_endpoint, runtime_model), + } if multi_endpoint_selection: settings = get_settings() identity_context = {'user_id': get_current_user_id()} diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 6b4d4a956..b0ea3f220 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -1457,7 +1457,8 @@ def upload_file(): extracted_content_raw, _extraction_engine_used, _extraction_engine_fallback = extract_content_with_extraction_engine( temp_file_path, extraction_mode=extraction_mode, - settings=settings + settings=settings, + is_image=is_image_file, ) # Convert pages_data list to string diff --git a/artifacts/cosmos_examples/cosmos-settings-example.json b/artifacts/cosmos_examples/cosmos-settings-example.json index 526e6ed71..434412928 100644 --- a/artifacts/cosmos_examples/cosmos-settings-example.json +++ b/artifacts/cosmos_examples/cosmos-settings-example.json @@ -39,7 +39,7 @@ }, "enable_image_generation": true, "azure_openai_image_gen_endpoint": "", - "azure_openai_image_gen_api_version": "2024-12-01-preview", + "azure_openai_image_gen_api_version": "2025-04-01-preview", "azure_openai_image_gen_authentication_type": "key", "azure_openai_image_gen_subscription_id": "", "azure_openai_image_gen_resource_group": "", diff --git a/docs/admin/ai-models.md b/docs/admin/ai-models.md index 1841a8011..5bc6892fc 100644 --- a/docs/admin/ai-models.md +++ b/docs/admin/ai-models.md @@ -148,7 +148,7 @@ The Image Generation section belongs to the Image Generation tab. Use it with th | Subscription ID | Defines behavior for the related admin workflow; verify the affected feature after saving. | Empty | `azure_openai_image_gen_subscription_id` | | Resource Group | Defines behavior for the related admin workflow; verify the affected feature after saving. | Empty | `azure_openai_image_gen_resource_group` | | Azure OpenAI Image Generation Key | Provides the secret credential used when the selected authentication mode requires one. | Empty | `azure_openai_image_gen_key` | -| Azure OpenAI Image Gen API Version | Pins the service API version SimpleChat sends with requests for this feature. | 2024-12-01-preview | `azure_openai_image_gen_api_version` | +| Azure OpenAI Image Gen API Version | Pins the service API version SimpleChat sends with requests for this feature. | 2025-04-01-preview | `azure_openai_image_gen_api_version` | | Azure APIM Endpoint | Provides the endpoint or route SimpleChat uses for this service. | Empty | `azure_apim_image_gen_endpoint` | | Azure APIM API Version | Pins the service API version SimpleChat sends with requests for this feature. | Empty | `azure_apim_image_gen_api_version` | | Azure APIM Deployment | Selects the deployment SimpleChat sends requests to for this capability. | Empty | `azure_apim_image_gen_deployment` | diff --git a/docs/admin/knowledge.md b/docs/admin/knowledge.md index 7198718aa..3660fa480 100644 --- a/docs/admin/knowledge.md +++ b/docs/admin/knowledge.md @@ -147,7 +147,24 @@ The Metadata Extraction section belongs to the Document Extraction tab. Use it w ### Multi-Modal Vision Analysis {#multimodal-vision-section} -The Multi-Modal Vision Analysis section belongs to the Document Extraction tab. Use it with the adjacent settings in this group so related rollout, access, and operational choices stay aligned. +Multi-modal vision analysis asks a vision-capable model to describe an uploaded image, list its +notable objects, and read any visible text. For workspace uploads, the description is added to the +image's search chunk and embedded with it, so an image can be found by what it shows rather than +only by the text OCR happens to read. When OCR finds no text at all, as with a photo of an empty +room, the description becomes the image's only chunk, and metadata extraction, when enabled, runs +on it. For images attached directly to a chat, the description is added to the file content the +model receives. + +Workspace images are analyzed only when Enhanced Citations is enabled. Images attached directly to +a chat are analyzed whenever this setting is on. + +The Vision Model setting stores a model name. When multi-endpoint models are enabled, SimpleChat +sends the analysis through the enabled AI connection that hosts an enabled model with that +deployment name, model name, or id, so a model deployed on a different connection than the primary +GPT resource works. A deployment name match wins over a model name match, and the first matching +connection in the list wins over later ones. When no connection matches, or multi-endpoint models +are off, the GPT connection is used, directly or through APIM. **Test Vision Analysis** uses the +same connection as ingestion. #### Settings @@ -167,7 +184,7 @@ The Multi-Modal Vision Analysis section belongs to the Document Extraction tab. | Content Understanding Key | Provides the secret credential used when the selected authentication mode requires one. | Empty | `azure_content_understanding_key` | | API Version | Default: | Not specified in defaults | `azure_content_understanding_api_version` | | Document Analyzer | Default: | Not specified in defaults | `azure_content_understanding_analyzer_id` | -| Image Analyzer | Default: | Not specified in defaults | `azure_content_understanding_image_analyzer_id` | +| Image Analyzer | Content Understanding analyzer that describes images embedded in DOCX and PPTX files, and standalone images when the document analyzer finds no text in them. Its `Summary` field is indexed as the image description. | prebuilt-imageSearch | `azure_content_understanding_image_analyzer_id` | | Analyze images embedded in DOCX and PPTX files | Exposes the capability after required services, permissions, and rollout policy are ready. | On | `enable_office_embedded_image_analysis`; capability toggle | | Minimum Image Size (pixels) | Images narrower or shorter than this are skipped as icons or spacers. | Not specified in defaults | `office_embedded_image_min_pixels` | | Maximum Images Per Document | Caps per-document cost. Duplicate images are analyzed once. | Not specified in defaults | `office_embedded_image_max_per_document` | diff --git a/docs/admin/workspaces.md b/docs/admin/workspaces.md index 8ee5fa721..ab96de416 100644 --- a/docs/admin/workspaces.md +++ b/docs/admin/workspaces.md @@ -63,7 +63,7 @@ The Public Workspaces section belongs to the Workspace Types tab. Use it with th | Enable Extract Meta Data | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_extract_meta_data`; capability toggle | | Extraction Model | Selects the deployment SimpleChat sends requests to for this capability. | Empty | `metadata_extraction_model` | | Enable Multi-Modal Vision Analysis | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_multimodal_vision`; capability toggle | -| Vision Model * | Select a GPT model with vision capabilities (for example, gpt-4o or supported GPT 5 and later models). Only vision-capable models are shown. | Empty | `multimodal_vision_model` | +| Vision Model * | Vision-capable model that describes uploaded images (for example, gpt-4o or supported GPT 5 and later models); only vision-capable models are shown. With multi-endpoint models enabled, it is called through the AI connection that hosts the selected model; otherwise through the GPT connection. | Empty | `multimodal_vision_model` | ## Files & Sharing {#files-sharing} diff --git a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md index 6b18429d3..355784d02 100644 --- a/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md +++ b/docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md @@ -45,6 +45,8 @@ flowchart TD F -->|Azure Public and configured| G[Content Understanding prebuilt-documentSearch] F -->|Gov / custom / unconfigured| J[Document Intelligence prebuilt-layout] G -->|Request fails| J + G -->|No content for an image| P[Content Understanding image analyzer] + P -->|No description| J H --> K[Extract embedded images from OOXML] K --> L{Active engine} @@ -156,6 +158,25 @@ Layout, which is the cheaper detector, and upgrades to Enhanced when the sample Images are always treated as Enhanced in Auto mode because they are single-page inputs that benefit most from figure and spatial analysis. +## Standalone Images With No Text + +`prebuilt-documentSearch` can return no pages for a photo that contains no text. When that happens +for an uploaded image, `extract_content_with_extraction_engine(..., is_image=True)` asks the image +analyzer (`azure_content_understanding_image_analyzer_id`, `prebuilt-imageSearch` by default) to +describe it, and indexes that description as page 1 with the reason *Content Understanding found no +text, so its image analyzer description was indexed*. If the image analyzer fails or returns +nothing, extraction falls back to Document Intelligence Layout as before. Workspace uploads and +images uploaded straight into a chat both use this path. PDFs and other files are not affected. + +Image analyzers return a placeholder such as `![image](pages/1)` as the markdown and put the +description in the `Summary` field. `analyze_image_with_content_understanding()` returns the +`Summary` value, any other non-empty string field as `Name: value`, any real markdown, and figure +descriptions, and ignores placeholder-only markdown. Images embedded in Office files use the same +function, so they are indexed with the description rather than the placeholder. + +Added in 0.261.047 for [#1583](https://github.com/microsoft/simplechat/issues/1583). See the +[Image Description Indexing Without OCR Text Fix](../fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md). + ## Images Inside Office Files Content Understanding accepts DOCX, DOC, and PPTX as input, but its figure analysis only applies to @@ -304,6 +325,7 @@ Enhanced extraction is disabled. | `functional_tests/test_office_embedded_image_extraction.py` | Real DOCX and PPTX zip round-trips: extraction, size and duplicate filtering, per-document cap, slide attribution, natural ordering, graceful handling of legacy and missing files, path-traversal defense against crafted entry names, and rejection of zip bombs and entity-expansion relationship parts | | `functional_tests/test_document_intelligence_pdf_image_extraction_mode.py` | Updated for the new engine split | | `functional_tests/test_document_intelligence_auto_reprocess_contract.py` | Updated for PDF and image extraction changes | +| `functional_tests/test_image_description_indexing_without_ocr_text.py` | Image analyzer `Summary` parsing, placeholder-only markdown, and the standalone image fallback before Document Intelligence Layout | ## Known Limitations diff --git a/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md new file mode 100644 index 000000000..2706f4bf5 --- /dev/null +++ b/docs/explanation/fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md @@ -0,0 +1,192 @@ +# Image Description Indexing Without OCR Text Fix (v0.261.047) + +Fixed in version: **0.261.047** + +Issue: [#1583](https://github.com/microsoft/simplechat/issues/1583) + +## Issue Description + +A user uploaded two nearly identical grayscale photos of an empty lab floor. Neither photo has +real text, and one is more overexposed. + +- The less-exposed photo was indexed and got a title. +- The overexposed photo ended with the status **Processing complete - no text found in image**, + no title, and 0 chunks in Azure AI Search, so chat could not use it. + +The AI vision model had described both photos, and the overexposed photo's vision analysis was +stored on its Cosmos DB document, but no chunk was ever written for it. + +A later upload of another image failed with `404 Not Found` from +`/openai/deployments/gpt-5.6-luna/chat/completions` on the primary Azure OpenAI resource. The +selected vision model was deployed only on a different AI connection, yet the admin +**Test Vision Analysis** button reported success. + +## Root Cause Analysis + +Four problems combined: + +1. **Only OCR text could create an image chunk.** `process_di_document()` built image chunks from + OCR pages only. Content Understanding `prebuilt-documentSearch` returned no pages for either + photo, so extraction fell back to Document Intelligence `prebuilt-layout`. Layout read a strip + of floor tape on the less-exposed photo as the page number `J` (confidence 0.18), producing the + markdown ``, and returned no content for the overexposed photo. The + bogus comment became the less-exposed photo's only chunk; the overexposed photo got none. + With `total_final_chunks_processed == 0`, final metadata extraction was skipped as well. +2. **The vector ignored the vision description.** `save_chunks()` appended the vision analysis to + the stored chunk text only after generating the embedding from the OCR text, so the logs showed + `embedding - 7 tokens` for the less-exposed photo's chunk. +3. **Content Understanding's image description was discarded.** The configured image analyzer, + `prebuilt-imageSearch`, described both photos well, but it returns the markdown + `![image](pages/1)` and puts the description in `contents[0].fields.Summary.valueString`. + `analyze_image_with_content_understanding()` ignored `fields`, so it only ever returned the + placeholder, and it was only used for images embedded in Office files. +4. **The vision model ignored its AI connection.** `analyze_image_with_vision_model()` always built + its client from the legacy `azure_openai_gpt_*` (or APIM) settings, while the admin test + resolved the model's configured connection. The setting stores only the model name + (`multimodal_vision_model`), so nothing mapped it back to the connection that hosts it. + +## Technical Details + +### Files Modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_documents.py` | Shared vision block formatting, enhanced text embedded in `save_chunks()` and `save_chunks_batch()`, description-only image chunks, `is_image` threading, vision model routing, shared model-endpoint client builder | +| `application/single_app/functions_content.py` | `extract_content_with_extraction_engine(..., is_image=False)` image analyzer fallback | +| `application/single_app/functions_content_understanding.py` | `analyze_image_with_content_understanding()` reads string fields and ignores placeholder-only markdown | +| `application/single_app/functions_model_endpoint_types.py` | Pure `find_enabled_model_endpoint_for_model_name()` lookup | +| `application/single_app/route_backend_settings.py` | Test Vision Analysis uses the same connection as ingestion | +| `application/single_app/route_frontend_chats.py` | Chat image uploads pass `is_image=is_image_file` | +| `application/single_app/config.py` | Version `0.261.047` | +| `functional_tests/test_image_description_indexing_without_ocr_text.py` | New regression coverage | +| `docs/admin/knowledge.md`, `docs/admin/workspaces.md` | How vision analysis is indexed and which connection it uses; Image Analyzer role | +| `docs/explanation/features/CONTENT_UNDERSTANDING_ENHANCED_EXTRACTION.md` | Standalone image analyzer fallback | +| `docs/explanation/release_notes.md` | Bug Fixes entry | + +### Code Changes Summary + +**Index image descriptions when OCR finds no text** (`functions_documents.py`) + +- `_format_vision_analysis_block(vision_analysis)` replaces the two copies of the inline + formatting. Valid analyses produce exactly the previous `=== AI Vision Analysis ===` block. It + returns `''` when the analysis is missing, not a dict, has an `error` key, or has no + description, objects, text, or analysis, so a vision failure message is never indexed. +- `_append_vision_block_to_chunk_text(chunk_text, vision_text)` appends the block, dropping its + leading blank lines when there is no OCR text. +- `save_chunks()` now builds the stored text before embedding and embeds that same text, keeping + the last-resort clamp guard. It skips, with a warning, any chunk that has nothing to index. + `save_chunks_batch()` uses the same helpers so both paths stay consistent. +- `process_di_document()` records whether this run produced a usable vision analysis + (`_is_usable_vision_analysis()`). For images it keeps only OCR pages with real text, treating a + page whose only content is Document Intelligence page-number or page-break annotations, such as + ``, as having none (`_has_indexable_ocr_text()`). Header and footer + comments, such as ``, still count as text because their quoted value + is the recognized text. When no OCR text remains and a usable analysis exists, it saves one + page-1 chunk flagged with `vision_description_only`, which `save_chunks()` fills with the stored + vision block. Final metadata extraction then runs as usual. Images with real OCR text are + unchanged. Vision analysis still runs only when Enhanced Citations is enabled. + +**Fall back to the Content Understanding image analyzer** (`functions_content_understanding.py`, +`functions_content.py`) + +- `analyze_image_with_content_understanding()` emits the `Summary` field as plain text and any + other non-empty string field as `Name: value`, ignores markdown made up only of image + placeholders such as `![image](pages/1)`, and keeps the existing figure-summary handling. This + also fixes images embedded in DOCX and PPTX files, which were indexed with just the placeholder + when Content Understanding analyzed them. +- `extract_content_with_extraction_engine()` takes `is_image`. When the document analyzer returns + no content for an image, it calls the image analyzer and returns its description as page 1 with + the reason *Content Understanding found no text, so its image analyzer description was indexed*. + If the image analyzer fails or returns nothing, it logs a warning and falls back to Document + Intelligence Layout exactly as before. PDFs and other files are unchanged. Workspace uploads + pass the flag through `_extract_pages_with_extraction_engine()`, and chat uploads pass it from + `route_frontend_chats.py`. + +**Route the vision model through its AI connection** (`functions_model_endpoint_types.py`, +`functions_documents.py`, `route_backend_settings.py`) + +- `find_enabled_model_endpoint_for_model_name(endpoints, model_name)` is a pure lookup over + enabled endpoints and enabled models. A deployment name or resolved request model wins over a + model name, which wins over a model id, and the first match in endpoint order wins within the + same precedence. It returns `(endpoint, model)` or `(None, None)`. +- `resolve_vision_model_endpoint(settings, vision_model)` applies that lookup to the normalized + `model_endpoints` when multi-endpoint models are enabled. +- `_build_model_endpoint_client_for_model(settings, endpoint_cfg, model_id, ...)` holds the + client construction that metadata extraction used: Key Vault secret resolution, model lookup, + request-model resolution, API version and protocol checks, the provider allowlist, and + `_build_model_endpoint_client()`. Metadata extraction calls it with the same order and error + messages as before. +- `analyze_image_with_vision_model()` uses the matched connection and sends + `model=`. Otherwise it builds the legacy APIM or direct client exactly as + before. It logs which connection was used by name and id only, and still returns `None` on + errors. Chat image uploads call the same function. +- `_test_multimodal_vision_connection()` resolves the model name the same way, so the admin test + exercises the connection ingestion will use, including when APIM is selected for GPT. + +## Testing + +`functional_tests/test_image_description_indexing_without_ocr_text.py` runs the pure lookup tests +in-process and runs the real modules in fresh normal and optimized processes with external I/O +blocked. It covers: + +- vision block formatting for valid, error, parse-failed, and empty analyses, +- `save_chunks()` storing and embedding the same text, the description-only chunk, skipped empty + chunks, and the unchanged clamp guard, plus `save_chunks_batch()` parity, +- the less-exposed (``), overexposed (empty page), and no-page image + cases, real OCR text, header-only OCR text, mixed pages, and failed, empty, missing, or disabled + vision analysis, +- the `prebuilt-imageSearch` payload, other string fields, placeholder-only markdown, and figure + de-duplication, +- the extraction fallback for images, analyzer failure and empty results, PDFs, document analyzer + content and failure, and Standard extraction, plus the chat upload call site, +- the image analyzer fallback when `functions_content_understanding` is imported before + `functions_content`, in a separate fresh process. The two modules already reach each other + through `functions_settings`, `functions_document_actions`, and `functions_search`, so both + import orders are checked, +- the vision connection lookup: match precedence, disabled endpoints and models, blank and + malformed input, legacy fallback with multi-endpoint off or no match, and APIM, +- unchanged metadata extraction client construction and every error message, +- the admin vision test using the ingestion connection with an APIM payload. + +Run it with: + +```powershell +python .\functional_tests\test_image_description_indexing_without_ocr_text.py +``` + +Related suites re-run: `test_content_understanding_extraction_engine.py`, +`test_office_embedded_image_extraction.py`, `test_figure_chunk_association.py`, +`test_multimodal_vision_multi_endpoint_connection.py`, +`test_document_intelligence_pdf_image_extraction_mode.py`, +`test_document_intelligence_auto_reprocess_contract.py`, +`test_privacy_logging_telemetry_audit.py`, the model endpoint provider suites, and the route policy +suites. + +## Impact + +- Text-free images are searchable through their AI vision description and get a title from + final metadata extraction. +- Image vectors now reflect the vision description, improving retrieval for image content. +- Existing documents are not re-indexed automatically. Re-upload an affected image to index it + with this fix. **Change Extraction** also re-indexes an image, but it does not re-run final + metadata extraction, so the image still has no generated title. +- A vision model name that exists on more than one enabled connection resolves by the precedence + above, because the setting stores only the name. +- `extract_content_with_extraction_engine()` still imports `functions_content_understanding` when + it runs, and that import is part of a pre-existing cycle: + `functions_content` → `functions_content_understanding` → `functions_settings` → + `functions_document_actions` → `functions_search` → `functions_content`. This fix adds + `analyze_image_with_content_understanding` to that existing import statement but no new module + dependency, so CodeQL reports the same `py/cyclic-import` finding at the statement's new + location. Breaking the cycle means changing the settings bootstrap chain and is left for separate + work. + +## Validation + +| Scenario | Before | After | +| --- | --- | --- | +| Overexposed photo, no OCR text | 0 chunks, no title, status "no text found in image" | 1 chunk holding the vision description, metadata extracted | +| Less-exposed photo, Layout reads `J` | Chunk text `` plus vision block, 7-token embedding | 1 chunk holding the vision description, embedded in full | +| Content Understanding image analyzer | `![image](pages/1)` | The `Summary` description | +| Vision model on another AI connection | 404 from the primary Azure OpenAI resource | Called on the connection that hosts it | +| Test Vision Analysis with APIM selected | Tested APIM while ingestion differed | Tests the connection ingestion uses | diff --git a/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md new file mode 100644 index 000000000..80a341509 --- /dev/null +++ b/docs/explanation/fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md @@ -0,0 +1,85 @@ +# Image Generation API Version Default Fix (v0.261.047) + +Fixed in version: **0.261.047** + +## Issue Description + +The Azure OpenAI image generation API version, `azure_openai_image_gen_api_version`, defaulted to +`2024-12-01-preview`. That version predates gpt-image model support, so a deployment that left the +default in place could not use gpt-image-1, gpt-image-1.5, or gpt-image-2 without an admin +changing it by hand. + +## Root Cause Analysis + +The shipped default, `2024-12-01-preview`, predates gpt-image model support; Microsoft documents +`2025-04-01-preview` for the gpt-image models. Raising the code-level default alone would not have +been enough: `get_settings()` merges defaults with `deep_merge_dicts()`, which only fills keys that +are missing from the persisted settings document. Every deployment that had already loaded +settings stored `2024-12-01-preview`, and would have kept it indefinitely. + +## Technical Details + +### Files Modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_settings.py` | New default constant and load-time migration | +| `docs/admin/ai-models.md` | Image Gen API Version default | +| `artifacts/cosmos_examples/cosmos-settings-example.json` | Image generation API version example | +| `application/external_apps/databaseseeder/artifacts/admin_settings.json` | Image generation API version seed value | +| `application/single_app/config.py` | Version `0.261.047` | +| `functional_tests/test_image_generation_api_version_default.py` | New regression coverage | +| `docs/explanation/release_notes.md` | Bug Fixes entry | + +### Code Changes Summary + +- `AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT = '2025-04-01-preview'` is the new default in + `get_settings()`. It is the newest dated Azure OpenAI API version and the one Microsoft documents + for gpt-image-1, gpt-image-1.5, and gpt-image-2. +- `AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT = '2024-12-01-preview'` records the old + default for the migration. +- `normalize_image_generation_api_version_settings(settings)` runs from + `normalize_loaded_settings()` on every settings load. It upgrades a stored value that is exactly + the previous default, or blank, to the new default and returns `True` so the change is + persisted. Any other value is a deliberate admin choice and is left untouched. +- `azure_apim_image_gen_api_version` is intentionally not migrated. An APIM gateway defines the API + version it accepts, which is why that setting has no default. + +A saved `2024-12-01-preview` cannot be told apart from the untouched old default, so that exact +value is always upgraded. Any other saved version is kept, so an admin can still pin a specific +version by entering it. + +## Testing + +`functional_tests/test_image_generation_api_version_default.py` covers: + +- the new default and the named constants, +- the previous default and padded previous default being upgraded, +- blank, whitespace, missing, and `None` values being upgraded, +- custom values such as `2025-03-01-preview`, and the new default itself, being left untouched, +- the APIM API version never changing, +- non-dict settings and non-string values being handled safely, +- the migration being wired into `normalize_loaded_settings()`, +- the admin docs and example artifacts showing the new default, +- a real `get_settings()` load, in fresh normal and optimized processes with external I/O blocked, + that migrates and persists the value to the settings document. + +Run it with: + +```powershell +python .\functional_tests\test_image_generation_api_version_default.py +``` + +The existing settings suites `test_app_settings_store_consistency.py`, +`test_settings_deep_merge_persistence_fix.py`, `test_tabular_parity_stale_settings_migration.py`, +and `test_app_settings_cache_versioning.py` were re-run and pass. + +## Validation + +| Stored `azure_openai_image_gen_api_version` | Before | After | +| --- | --- | --- | +| `2024-12-01-preview` | Kept | `2025-04-01-preview` | +| Blank | Kept blank | `2025-04-01-preview` | +| `2025-03-01-preview` or any other value | Kept | Kept | +| New deployment | `2024-12-01-preview` | `2025-04-01-preview` | +| `azure_apim_image_gen_api_version` | Kept | Kept | diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index fa1093ec9..7e798ad1f 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,23 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.047)** + +#### Bug Fixes + +* **Images Without OCR Text Are Now Indexed From Their Vision Description** + * Fixed text-free images, such as a photo of an empty room, ending with no search chunk, no title, and nothing chat could use, even though the AI vision model had described them. When OCR finds no text, the vision description is now indexed as the image's chunk, so the image is searchable and final metadata extraction runs as usual. + * Document Intelligence Layout output that is only page-number or page-break annotations, such as a stray `` read from floor tape, no longer counts as image text. Header and footer text still does. Failed or empty vision analyses are never indexed. + * Image chunks are now embedded from the same text that is stored, including the vision description, instead of from the OCR text alone. Images with real OCR text keep their existing chunk. + * With Enhanced extraction, a standalone image that the Content Understanding document analyzer returns nothing for is now described by the Content Understanding image analyzer before falling back to Document Intelligence Layout. This applies to workspace uploads and to images uploaded straight into a chat. The image analyzer's `Summary` field is now read, which also replaces the `![image](pages/1)` placeholder that images embedded in DOCX and PPTX files were indexed with. + * The multi-modal vision model is now called through the AI connection that hosts it when multi-endpoint models are enabled, instead of always using the legacy GPT connection, which returned 404 for models deployed on another connection. The Test Vision Analysis button now uses the same connection as ingestion. + * (Ref: #1583, `functions_documents.py`, `functions_content.py`, `functions_content_understanding.py`, `functions_model_endpoint_types.py`, `route_backend_settings.py`, `route_frontend_chats.py`, [Image Description Indexing Without OCR Text Fix](fixes/IMAGE_DESCRIPTION_INDEXING_WITHOUT_OCR_TEXT_FIX.md)) + +* **Image Generation Uses The gpt-image API Version By Default** + * The default Azure OpenAI image generation API version is now `2025-04-01-preview`, the newest dated Azure OpenAI API version and the one documented for gpt-image-1, gpt-image-1.5, and gpt-image-2. The previous `2024-12-01-preview` default predates gpt-image support. + * Existing deployments still storing the previous default, or a blank value, are upgraded automatically when settings load. Any other saved version is kept, and the APIM image generation API version is never changed because the gateway defines it. + * (Ref: `functions_settings.py`, `azure_openai_image_gen_api_version`, [Image Generation API Version Default Fix](fixes/IMAGE_GENERATION_API_VERSION_DEFAULT_FIX.md)) + ### **(v0.261.046)** #### Bug Fixes diff --git a/functional_tests/test_image_description_indexing_without_ocr_text.py b/functional_tests/test_image_description_indexing_without_ocr_text.py new file mode 100644 index 000000000..30d2edfe9 --- /dev/null +++ b/functional_tests/test_image_description_indexing_without_ocr_text.py @@ -0,0 +1,1219 @@ +#!/usr/bin/env python3 +# test_image_description_indexing_without_ocr_text.py +""" +Functional test for indexing image descriptions when OCR finds no text. +Version: 0.261.047 +Implemented in: 0.261.047 + +Issue #1583: two nearly identical photos of an empty lab floor had no real text. Document +Intelligence Layout returned only a bogus page-number comment for one and nothing for the other, +so the second image was never indexed and its AI vision description was unreachable from chat. +Content Understanding's image analyzer described both photos, but its description was ignored, +and the vision model was always called on the legacy GPT connection even when it was hosted on a +different AI connection. + +This test ensures that: + - the vision block keeps its existing format and is empty for failed or empty analyses, + - save_chunks and save_chunks_batch embed the same text they store, including the vision block, + and never write a chunk with nothing to index, + - an image whose OCR output is empty, or only Document Intelligence page-number annotations, is + indexed from its vision description, while images with real OCR text, including header and + footer text, are unchanged, + - the Content Understanding image analyzer's Summary field is read, and extraction falls back + to it for standalone images before Document Intelligence Layout, both for workspace uploads + and for images uploaded straight into a chat, + - the stored vision model name resolves to the enabled AI connection that hosts it, falling + back to the legacy GPT connection otherwise, and the admin test uses the same connection, + - metadata extraction builds its client exactly as before, with the same error messages. + +The model-name lookup and the chat upload call-site check run in-process. Checks that need the +real application modules run in fresh normal and optimized processes with external I/O blocked, +stubbing only the Azure-facing calls. A separate fresh process imports functions_content_understanding +before functions_content, the reverse of the main probe, because the two modules reach each other +through a pre-existing import cycle, and checks that the image analyzer fallback still resolves. +""" + +import ast +import copy +from contextlib import ExitStack +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +TEST_ROOT = ROOT / "functional_tests" +IMPLEMENTED_IN_VERSION = "0.261.047" + +for _path in (str(TEST_ROOT), str(APP_ROOT)): + if _path not in sys.path: + sys.path.insert(0, _path) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +# A 1x1 PNG, enough for code that only reads and base64-encodes the file. +TEST_PNG_BYTES = bytes.fromhex( + "89504e470d0a1a0a0000000d4948445200000001000000010806000000" + "1f15c4890000000d49444154789c63f8cfc0f01f0005000201e1c5a8e1" + "0000000049454e44ae426082" +) + +# Layout output for the less-exposed photo: a strip of floor tape read as the page number "J". +LESS_EXPOSED_DI_PAGES = [{"page_number": 1, "content": ''}] +# Layout output for the overexposed photo: one page with no content. +OVEREXPOSED_DI_PAGES = [{"page_number": 1, "content": ""}] + +CU_IMAGE_SUMMARY = ( + "The image shows a mostly empty indoor space with a tiled floor. In the background, there are " + "several checkerboard calibration patterns placed against the wall." +) +CU_IMAGE_SEARCH_RESULT = { + "analyzerId": "prebuilt-imageSearch", + "apiVersion": "2025-11-01", + "contents": [ + { + "kind": "document", + "markdown": "![image](pages/1)\n", + "fields": { + "Summary": {"type": "string", "valueString": CU_IMAGE_SUMMARY}, + }, + } + ], +} + +VISION_ANALYSIS = { + "description": "A grayscale photo of an empty laboratory floor with checkerboard calibration targets on the far wall.", + "objects": ["tiled floor", "checkerboard calibration target", "wall"], + "text": "", + "analysis": "The room looks like a camera calibration area in a robotics lab.", + "model": "gpt-5.4", +} +VISION_RESPONSE_JSON = { + "description": VISION_ANALYSIS["description"], + "objects": VISION_ANALYSIS["objects"], + "text": "", + "analysis": VISION_ANALYSIS["analysis"], +} + +CU_SETTINGS = { + "azure_content_understanding_endpoint": "https://content-understanding.invalid", + "azure_content_understanding_key": "content-understanding-test-key", + "azure_content_understanding_authentication_type": "key", +} + +PRIMARY_ENDPOINT_SECRET = "primary-endpoint-secret-must-not-be-logged" +GLOBAL_ENDPOINT_SECRET = "global-endpoint-secret-must-not-be-logged" + + +def _require(condition, message): + """Raise explicitly so checks survive python -O.""" + if not condition: + raise AssertionError(message) + + +def build_model_endpoints(): + """Two AI connections: the primary one, and 'Global', which alone hosts gpt-5.6-luna.""" + return [ + { + "id": "primary-endpoint", + "name": "Primary", + "provider": "aoai", + "enabled": True, + "connection": { + "endpoint": "https://simplechat-openai.invalid", + "openai_api_version": "2024-12-01-preview", + }, + "auth": {"type": "api_key", "api_key": PRIMARY_ENDPOINT_SECRET}, + "models": [ + {"id": "gpt-4o-model", "deploymentName": "gpt-4o", "modelName": "gpt-4o", "enabled": True}, + ], + }, + { + "id": "global-endpoint", + "name": "Global", + "provider": "aoai", + "enabled": True, + "connection": { + "endpoint": "https://aoai-global-team.invalid", + "openai_api_version": "2025-04-01-preview", + }, + "auth": {"type": "api_key", "api_key": GLOBAL_ENDPOINT_SECRET}, + "models": [ + { + "id": "luna-model", + "deploymentName": "luna-prod", + "modelName": "gpt-5.6-luna", + "displayName": "GPT-5.6 Luna", + "enabled": True, + }, + ], + }, + ] + + +def legacy_vision_block(vision_analysis): + """The inline formatting save_chunks used before this fix, kept here for format parity.""" + vision_text_parts = [] + vision_text_parts.append("\n\n=== AI Vision Analysis ===") + vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") + if vision_analysis.get('description'): + vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") + if vision_analysis.get('objects'): + objects_list = vision_analysis['objects'] + if isinstance(objects_list, list): + vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") + else: + vision_text_parts.append(f"\nObjects Detected: {objects_list}") + if vision_analysis.get('text'): + vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") + if vision_analysis.get('analysis'): + vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") + return "\n".join(vision_text_parts) + + +class FakeChatClient: + """Records chat completion requests and answers with a canned vision analysis.""" + + def __init__(self, content): + self.requests = [] + self._content = content + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) + + def _create(self, **params): + self.requests.append(params) + return SimpleNamespace( + id="chatcmpl-test", + model=params.get("model"), + usage=SimpleNamespace(prompt_tokens=10, completion_tokens=20, total_tokens=30), + choices=[ + SimpleNamespace( + finish_reason="stop", + message=SimpleNamespace(content=self._content, refusal=None), + ) + ], + ) + + +def make_recording_azure_openai(records): + """Return an AzureOpenAI stand-in that records its constructor arguments.""" + + class RecordingAzureOpenAI(FakeChatClient): + def __init__(self, **kwargs): + super().__init__(json.dumps(VISION_RESPONSE_JSON)) + self.init_kwargs = kwargs + records.append(self) + + return RecordingAzureOpenAI + + +def make_recording_client_builder(records, client=None): + """Return a build_model_endpoint_sync_chat_client stand-in that records its arguments.""" + + def fake_build(*args, **kwargs): + built_client = client or FakeChatClient(json.dumps(VISION_RESPONSE_JSON)) + records.append({"args": args, "kwargs": kwargs, "client": built_client}) + return built_client, "azure_openai" + + return fake_build + + +def write_test_image(directory, file_name="lab-floor.png"): + image_path = os.path.join(directory, file_name) + with open(image_path, "wb") as image_file: + image_file.write(TEST_PNG_BYTES) + return image_path + + +# --------------------------------------------------------------------------- +# In-process checks of the pure model-name lookup +# --------------------------------------------------------------------------- + +def _load_model_endpoint_types(): + import functions_model_endpoint_types # noqa: PLC0415 - loaded after sys.path is prepared + + return functions_model_endpoint_types + + +def test_model_name_lookup_matches_deployment_model_name_and_id(): + """A stored name matches a deployment, the request model, a model name, or a model id.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = build_model_endpoints() + + for stored_name in ("luna-prod", "gpt-5.6-luna", "luna-model", " gpt-5.6-luna "): + endpoint, model = find(endpoints, stored_name) + assert endpoint is endpoints[1], f"{stored_name!r} should resolve to the Global connection" + assert model is endpoints[1]["models"][0], f"{stored_name!r} should resolve to the Luna model" + + endpoint, model = find(endpoints, "gpt-4o") + assert endpoint is endpoints[0] and model["id"] == "gpt-4o-model" + + # OpenAI-style Custom connections send the model name, so it is their request model. + custom_endpoints = [{ + "id": "custom-endpoint", + "provider": "custom", + "api_type": "openai", + "enabled": True, + "models": [{"id": "llama-model", "modelName": "llama-3.2-vision", "enabled": True}], + }] + endpoint, model = find(custom_endpoints, "llama-3.2-vision") + assert endpoint is custom_endpoints[0] and model["id"] == "llama-model" + + +def test_model_name_lookup_prefers_deployments_then_first_endpoint(): + """Deployment and request-model matches beat model names; ties go to the first endpoint.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = [ + { + "id": "family-endpoint", + "enabled": True, + "models": [{"id": "family-model", "deploymentName": "gpt-4o-prod", "modelName": "gpt-4o", "enabled": True}], + }, + { + "id": "deployment-endpoint", + "enabled": True, + "models": [{"id": "deployment-model", "deploymentName": "gpt-4o", "enabled": True}], + }, + { + "id": "second-deployment-endpoint", + "enabled": True, + "models": [{"id": "second-deployment-model", "deploymentName": "gpt-4o", "enabled": True}], + }, + ] + + endpoint, model = find(endpoints, "gpt-4o") + assert endpoint["id"] == "deployment-endpoint", "A deployment match must beat an earlier model-name match." + assert model["id"] == "deployment-model", "The first deployment match in endpoint order must win." + + endpoints_by_name = [ + {"id": "first", "enabled": True, "models": [{"id": "a", "deploymentName": "a-prod", "modelName": "shared", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [{"id": "b", "deploymentName": "b-prod", "modelName": "shared", "enabled": True}]}, + ] + endpoint, _ = find(endpoints_by_name, "shared") + assert endpoint["id"] == "first", "Equal model-name matches must resolve to the first endpoint." + + endpoints_by_id = [ + {"id": "first", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "x-prod", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [ + {"id": "other", "deploymentName": "other-prod", "modelName": "shared-id", "enabled": True}, + ]}, + ] + endpoint, _ = find(endpoints_by_id, "shared-id") + assert endpoint["id"] == "second", "A model-name match must beat an earlier model-id match." + + endpoints_only_id = [ + {"id": "first", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "x-prod", "enabled": True}]}, + {"id": "second", "enabled": True, "models": [{"id": "shared-id", "deploymentName": "y-prod", "enabled": True}]}, + ] + endpoint, _ = find(endpoints_only_id, "shared-id") + assert endpoint["id"] == "first", "Equal model-id matches must resolve to the first endpoint." + + +def test_model_name_lookup_ignores_disabled_endpoints_and_models(): + """Disabled endpoints and disabled models never match.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + assert find(disabled_endpoint, "gpt-5.6-luna") == (None, None) + + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + assert find(disabled_model, "luna-prod") == (None, None) + + fallback = [ + {"id": "off", "enabled": False, "models": [{"id": "m1", "deploymentName": "vision", "enabled": True}]}, + {"id": "on", "enabled": True, "models": [ + {"id": "m2", "deploymentName": "vision", "enabled": False}, + {"id": "m3", "deploymentName": "vision", "enabled": True}, + ]}, + ] + endpoint, model = find(fallback, "vision") + assert endpoint["id"] == "on" and model["id"] == "m3" + + +def test_model_name_lookup_handles_blank_and_malformed_input(): + """Blank names and malformed endpoint data return (None, None) instead of raising.""" + find = _load_model_endpoint_types().find_enabled_model_endpoint_for_model_name + endpoints = build_model_endpoints() + + for model_name in (None, "", " "): + assert find(endpoints, model_name) == (None, None) + for malformed in (None, {}, "gpt-4o", 42): + assert find(malformed, "gpt-4o") == (None, None) + assert find([None, "text", {"id": "no-models", "enabled": True}, {"models": "gpt-4o"}], "gpt-4o") == (None, None) + assert find([{"id": "e", "models": [None, "gpt-4o", {"deploymentName": "gpt-4o"}]}], "gpt-4o")[1] == {"deploymentName": "gpt-4o"} + assert find(endpoints, "not-configured") == (None, None) + + before = copy.deepcopy(endpoints) + find(endpoints, "gpt-5.6-luna") + assert endpoints == before, "The lookup must not mutate the endpoint records." + + +def test_version_is_at_least_implementation_version(): + """The app version must be at or beyond the version this fix shipped in.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + + +def _call_name(call_node): + if isinstance(call_node.func, ast.Name): + return call_node.func.id + if isinstance(call_node.func, ast.Attribute): + return call_node.func.attr + return "" + + +def test_chat_uploads_request_the_image_analyzer_fallback(): + """Images uploaded straight into a chat must get the same image-analyzer fallback.""" + tree = ast.parse((APP_ROOT / "route_frontend_chats.py").read_text(encoding="utf-8-sig")) + calls = [ + node for node in ast.walk(tree) + if isinstance(node, ast.Call) and _call_name(node) == "extract_content_with_extraction_engine" + ] + assert len(calls) == 1, f"Expected one chat upload extraction call site, found {len(calls)}." + + keywords = {keyword.arg: keyword.value for keyword in calls[0].keywords} + assert isinstance(keywords.get("is_image"), ast.Name) and keywords["is_image"].id == "is_image_file", ( + "The chat upload route must pass is_image=is_image_file to extract_content_with_extraction_engine()." + ) + + enclosing_function = min( + ( + node for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.lineno <= calls[0].lineno <= node.end_lineno + ), + key=lambda node: node.end_lineno - node.lineno, + ) + assert any( + isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "is_image_file" for target in node.targets) + for node in ast.walk(enclosing_function) + ), "is_image_file must be computed in the same chat upload function." + + +# --------------------------------------------------------------------------- +# Real-module checks, run by the offline probe in a fresh process +# --------------------------------------------------------------------------- + +def check_vision_block_formatting(documents): + print("Checking vision block formatting...") + block = documents._format_vision_analysis_block(VISION_ANALYSIS) + _require(block == legacy_vision_block(VISION_ANALYSIS), "A valid analysis must keep the existing vision block format.") + _require(block.startswith("\n\n=== AI Vision Analysis ==="), "The block must still start with the section header.") + + string_objects = {**VISION_ANALYSIS, "objects": "tiled floor, wall"} + _require( + documents._format_vision_analysis_block(string_objects) == legacy_vision_block(string_objects), + "A non-list objects value must keep the existing format.", + ) + + # json.loads fallback analyses carry the raw response as a description and no error key. + parse_failed = {"description": "Raw model text", "raw_response": "Raw model text", "parse_failed": True, "model": "gpt-5.4"} + _require("Description: Raw model text" in documents._format_vision_analysis_block(parse_failed), + "An analysis whose JSON could not be parsed still has a usable description.") + + unusable = [ + None, + "not a dict", + {}, + {"model": "gpt-5.4"}, + {"description": " ", "objects": [], "text": "", "analysis": None, "model": "gpt-5.4"}, + {"description": "Model returned empty content with no refusal message", + "error": "Model returned empty content with no refusal message", "model": "gpt-5.4", "parse_failed": True}, + {"error": ""}, + ] + for analysis in unusable: + _require(documents._format_vision_analysis_block(analysis) == "", + f"Unusable analysis must format to an empty string: {analysis!r}") + _require(not documents._is_usable_vision_analysis(analysis), f"Analysis must be unusable: {analysis!r}") + _require(documents._is_usable_vision_analysis({"objects": ["floor"]}), "Objects alone make an analysis usable.") + + append = documents._append_vision_block_to_chunk_text + _require(append("EXIT 12", block) == "EXIT 12" + block, "OCR text must be followed by the unchanged block.") + _require(append("", block) == block.lstrip(), "Without OCR text the block must not keep leading blank lines.") + _require(append(" \n", block).startswith("=== AI Vision Analysis ==="), "Whitespace-only text counts as no text.") + _require(append(None, "") == "" and append("EXIT 12", "") == "EXIT 12", "No block leaves the text unchanged.") + + +def check_comment_only_ocr_is_not_text(documents): + print("Checking Document Intelligence page-number-only OCR detection...") + has_text = documents._has_indexable_ocr_text + for content in ( + '', + '\n\n\n', + "", + "", + None, + " \n ", + ): + _require(not has_text(content), f"Content must count as no text: {content!r}") + for content in ( + '\nEXIT 12', + # Header and footer comments carry recognized text, so they must keep the image indexed. + '', + '\n', + "Selection marks detected:\n- Door closed: checked", + "J", + ): + _require(has_text(content), f"Content must count as real text: {content!r}") + + +def _run_save_chunks(documents, page_text, vision_analysis, max_characters=100000): + embedded_inputs = [] + uploaded_documents = [] + metadata = {"id": "doc-1583", "version": 1, "title": "", "tags": [], "shared_user_ids": []} + if vision_analysis is not None: + metadata["vision_analysis"] = vision_analysis + + def fake_generate_embedding(text): + embedded_inputs.append(text) + return [0.25, 0.5], {"total_tokens": 42, "prompt_tokens": 42, "model_deployment_name": "text-embedding-3-small"} + + def fake_search_write(search_client, operation_name, *args, **kwargs): + uploaded_documents.extend(kwargs.get("documents") or []) + return [{"succeeded": True}] + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: copy.deepcopy(metadata))) + stack.enter_context(patch.object(documents, "generate_embedding", fake_generate_embedding)) + stack.enter_context(patch.object(documents, "get_embedding_safe_chunk_characters", lambda *a, **k: max_characters)) + stack.enter_context(patch.object(documents, "_execute_document_search_write", fake_search_write)) + stack.enter_context(patch.object(documents, "add_file_task_to_file_processing_log", lambda **_: None)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + stack.enter_context(patch.dict(documents.CLIENTS, {"search_client_user": object()})) + token_usage = documents.save_chunks( + page_text_content=page_text, + page_number=1, + file_name="lab-floor.png", + user_id="user-1", + document_id="doc-1583", + ) + return token_usage, embedded_inputs, uploaded_documents + + +def check_save_chunks_embeds_the_stored_text(documents): + print("Checking save_chunks embeds and stores the vision description...") + block = legacy_vision_block(VISION_ANALYSIS) + + token_usage, embedded, uploaded = _run_save_chunks(documents, "", VISION_ANALYSIS) + _require(len(uploaded) == 1, "A description-only image chunk must be written.") + chunk_text = uploaded[0]["chunk_text"] + _require(chunk_text == block.lstrip(), "The description-only chunk must hold exactly the vision block.") + _require(chunk_text.startswith("=== AI Vision Analysis ==="), "The chunk must not start with blank lines.") + _require(VISION_ANALYSIS["description"] in chunk_text, "The chunk must contain the vision description.") + _require(embedded == [chunk_text], "The embedding must be generated from the stored chunk text.") + _require(uploaded[0]["id"] == "doc-1583_1" and uploaded[0]["page_number"] == 1, "The chunk must be page 1.") + _require(token_usage and token_usage["total_tokens"] == 42, "Token usage must still be returned.") + + _, embedded, uploaded = _run_save_chunks(documents, "EXIT 12", VISION_ANALYSIS) + _require(uploaded[0]["chunk_text"] == "EXIT 12" + block, "OCR chunks must keep text followed by the vision block.") + _require(embedded == ["EXIT 12" + block], "The vector must reflect the description, not only the OCR text.") + + _, embedded, uploaded = _run_save_chunks(documents, "Quarterly report text.", None) + _require(uploaded[0]["chunk_text"] == "Quarterly report text.", "Chunks without vision analysis are unchanged.") + _require(embedded == ["Quarterly report text."], "Embeddings without vision analysis are unchanged.") + + error_analysis = {"description": "Model refused to respond", "error": "Model refused to respond", "model": "gpt-5.4"} + for page_text, analysis in (("", error_analysis), ("", None), (" \n", {"model": "gpt-5.4"})): + token_usage, embedded, uploaded = _run_save_chunks(documents, page_text, analysis) + _require(token_usage is None and not embedded and not uploaded, + f"A chunk with nothing to index must not be embedded or written: {page_text!r}, {analysis!r}") + + _, embedded, uploaded = _run_save_chunks(documents, "", error_analysis.copy() | {"description": ""}, max_characters=100000) + _require(not uploaded, "A failed vision analysis must never be indexed.") + + long_text = "OCR " * 40 + _, embedded, uploaded = _run_save_chunks(documents, long_text, VISION_ANALYSIS, max_characters=60) + _require(len(embedded[0]) == 60 and embedded[0] == (long_text + block)[:60], + "The last-resort guard must still clamp only the embedding input.") + _require(uploaded[0]["chunk_text"] == long_text + block, "The stored chunk text must never be clamped.") + + +def check_save_chunks_batch_matches_save_chunks(documents, content): + print("Checking save_chunks_batch embeds the same text it stores...") + block = legacy_vision_block(VISION_ANALYSIS) + batched_inputs = [] + uploaded_documents = [] + + def fake_batch(texts): + batched_inputs.append(list(texts)) + return [([0.1, 0.2], {"total_tokens": 5, "prompt_tokens": 5, "model_deployment_name": "text-embedding-3-small"}) for _ in texts] + + def fake_search_write(search_client, operation_name, *args, **kwargs): + uploaded_documents.extend(kwargs.get("documents") or []) + return [{"succeeded": True}] + + def run(metadata, chunks): + batched_inputs.clear() + uploaded_documents.clear() + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: copy.deepcopy(metadata))) + stack.enter_context(patch.object(content, "generate_embeddings_batch", fake_batch)) + stack.enter_context(patch.object(documents, "_execute_document_search_write", fake_search_write)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + stack.enter_context(patch.dict(documents.CLIENTS, {"search_client_user": object()})) + return documents.save_chunks_batch(chunks, "user-1", "doc-1583") + + chunks = [ + {"page_text_content": "Page one", "page_number": 1, "file_name": "lab-floor.png"}, + {"page_text_content": "", "page_number": 2, "file_name": "lab-floor.png"}, + ] + usage = run({"version": 1, "vision_analysis": VISION_ANALYSIS}, chunks) + stored = [document["chunk_text"] for document in uploaded_documents] + _require(stored == ["Page one" + block, block.lstrip()], "Batch chunks must use the shared vision formatting.") + _require(batched_inputs == [stored], "Batch embeddings must be generated from the stored chunk text.") + _require(usage["total_tokens"] == 10, "Batch token usage must still be accumulated.") + + usage = run({"version": 1}, chunks) + stored = [document["chunk_text"] for document in uploaded_documents] + _require(stored == ["Page one"] and batched_inputs == [["Page one"]], + "Without a vision analysis, empty batch chunks are skipped and others are unchanged.") + _require(usage["total_tokens"] == 5, "Skipped chunks must not add embedding token usage.") + + usage = run({"version": 1}, [{"page_text_content": " ", "page_number": 1, "file_name": "x.md"}]) + _require(not batched_inputs and not uploaded_documents and usage["total_tokens"] == 0, + "A batch with nothing to index must not call the embedding endpoint.") + + +def _run_image_ingestion(documents, image_path, di_pages, vision_result, enable_vision=True): + saved_chunks = [] + extraction_calls = [] + status_updates = [] + vision_calls = [] + settings = { + "enable_multimodal_vision": enable_vision, + "enable_extract_meta_data": False, + "enable_enhanced_extraction": True, + "document_intelligence_pdf_image_extraction_mode": "layout", + } + + def fake_extract(file_path, extraction_mode, extraction_engine, settings=None, pages=None, is_image=False): + extraction_calls.append({"extraction_mode": extraction_mode, "extraction_engine": extraction_engine, "is_image": is_image}) + return copy.deepcopy(di_pages), "document_intelligence", "Content Understanding returned no content, so Document Intelligence Layout was used" + + def fake_vision(*args, **kwargs): + vision_calls.append(args) + return copy.deepcopy(vision_result) + + def fake_save_chunks(**kwargs): + saved_chunks.append(kwargs) + return {"total_tokens": 7, "prompt_tokens": 7, "model_deployment_name": "text-embedding-3-small"} + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "get_settings", lambda: dict(settings))) + stack.enter_context(patch.object(documents, "get_chunk_size_config", lambda *a, **k: {})) + stack.enter_context(patch.object(documents, "is_enhanced_extraction_enabled", lambda *a, **k: True)) + stack.enter_context(patch.object(documents, "get_effective_document_intelligence_pdf_image_extraction_mode", lambda *a, **k: "layout")) + stack.enter_context(patch.object(documents, "get_document_intelligence_auto_sample_pages", lambda *a, **k: 3)) + stack.enter_context(patch.object(documents, "_resolve_extraction_engine_for_mode", lambda *a, **k: ("content_understanding", ""))) + stack.enter_context(patch.object(documents, "upload_to_blob", lambda **_: None)) + stack.enter_context(patch.object(documents, "_extract_pages_with_extraction_engine", fake_extract)) + stack.enter_context(patch.object(documents, "analyze_image_with_vision_model", fake_vision)) + stack.enter_context(patch.object(documents, "get_document_metadata", lambda **_: {"number_of_pages": 1})) + stack.enter_context(patch.object(documents, "save_chunks", fake_save_chunks)) + stack.enter_context(patch.object(documents, "log_event", lambda *a, **k: None)) + result = documents.process_di_document( + document_id="doc-1583", + user_id="user-1", + temp_file_path=image_path, + original_filename="lab-floor.png", + file_ext=".png", + enable_enhanced_citations=True, + update_callback=lambda **kwargs: status_updates.append(kwargs), + auto_extract_metadata=False, + ) + return result, saved_chunks, extraction_calls, status_updates, vision_calls + + +def check_text_free_images_are_indexed_from_vision(documents, work_dir): + print("Checking text-free images are indexed from their vision description...") + image_path = write_test_image(work_dir) + description_only_chunk = { + "page_text_content": "", + "page_number": 1, + "file_name": "lab-floor.png", + "user_id": "user-1", + "document_id": "doc-1583", + } + + for label, di_pages in ( + ("less-exposed photo (comment-only Layout output)", LESS_EXPOSED_DI_PAGES), + ("overexposed photo (empty Layout page)", OVEREXPOSED_DI_PAGES), + ("no pages at all", []), + ): + result, saved, extraction_calls, statuses, _ = _run_image_ingestion(documents, image_path, di_pages, VISION_ANALYSIS) + _require(result[0] == 1, f"{label}: exactly one chunk must be saved, got {result[0]}.") + _require(saved == [description_only_chunk], f"{label}: expected one description-only page-1 chunk, got {saved}.") + _require(result[1] == 7 and result[2] == "text-embedding-3-small", f"{label}: token usage must be accumulated.") + _require(extraction_calls and extraction_calls[0]["is_image"] is True, f"{label}: is_image must reach extraction.") + _require(any("indexing its AI vision description" in str(update.get("status")) for update in statuses), + f"{label}: the status must say the vision description was indexed.") + + result, saved, _, _, _ = _run_image_ingestion( + documents, image_path, [{"page_number": 1, "content": "EXIT 12"}], VISION_ANALYSIS + ) + _require(result[0] == 1 and saved[0]["page_text_content"] == "EXIT 12", + "Images with real OCR text must keep their OCR chunk unchanged.") + + header_only_pages = [{"page_number": 1, "content": ''}] + for vision_result in (VISION_ANALYSIS, None): + result, saved, _, _, _ = _run_image_ingestion(documents, image_path, header_only_pages, vision_result) + _require(result[0] == 1 and saved[0]["page_text_content"] == header_only_pages[0]["content"], + "Text Layout reports as a page header is real OCR text and must stay indexed.") + + result, saved, _, _, _ = _run_image_ingestion( + documents, + image_path, + [{"page_number": 1, "content": ''}, {"page_number": 2, "content": "Second page text"}], + VISION_ANALYSIS, + ) + _require(result[0] == 1 and [(c["page_number"], c["page_text_content"]) for c in saved] == [(2, "Second page text")], + "Only OCR pages with real text may be saved.") + + failed_analysis = {"description": "Model returned empty content with no refusal message", + "error": "Model returned empty content with no refusal message", "model": "gpt-5.4", "parse_failed": True} + for label, vision_result, enable_vision in ( + ("failed vision analysis", failed_analysis, True), + ("empty vision analysis", {"description": "", "objects": [], "text": "", "analysis": "", "model": "gpt-5.4"}, True), + ("no vision analysis", None, True), + ("vision disabled", VISION_ANALYSIS, False), + ): + result, saved, _, _, vision_calls = _run_image_ingestion( + documents, image_path, LESS_EXPOSED_DI_PAGES, vision_result, enable_vision=enable_vision + ) + _require(result[0] == 0 and not saved, f"{label}: a text-free image with no usable description must save nothing.") + if not enable_vision: + _require(not vision_calls, "Vision analysis must not run when it is disabled.") + + +def check_content_understanding_image_analyzer(content_understanding, work_dir): + print("Checking the Content Understanding image analyzer parsing...") + image_path = write_test_image(work_dir, "cu-image.png") + analyzer_calls = [] + + def analyze(result): + analyzer_calls.clear() + + def fake_analyze_file(file_path, analyzer_id=None, page_range=None, settings=None, config_override=None, max_wait_seconds=None): + analyzer_calls.append(analyzer_id) + return copy.deepcopy(result) + + with patch.object(content_understanding, "analyze_file_with_content_understanding", fake_analyze_file): + return content_understanding.analyze_image_with_content_understanding(image_path, settings=dict(CU_SETTINGS)) + + text = analyze(CU_IMAGE_SEARCH_RESULT) + _require(text == CU_IMAGE_SUMMARY, f"The Summary field must be returned as plain text, got {text!r}.") + _require(analyzer_calls == ["prebuilt-imageSearch"], f"The configured image analyzer must be used: {analyzer_calls}.") + + with_fields = copy.deepcopy(CU_IMAGE_SEARCH_RESULT) + with_fields["contents"][0]["fields"].update({ + "Setting": {"type": "string", "valueString": "Robotics calibration lab"}, + "ObjectCount": {"type": "number", "valueNumber": 4}, + "Caption": {"type": "string", "valueString": " "}, + "Tags": {"type": "array", "valueArray": [{"type": "string", "valueString": "floor"}]}, + }) + text = analyze(with_fields) + _require(text == f"{CU_IMAGE_SUMMARY}\n\nSetting: Robotics calibration lab", + f"Other string fields must be emitted as 'Name: value' after the summary, got {text!r}.") + + only_placeholder = {"contents": [{"kind": "document", "markdown": "![image](pages/1)\n![image](pages/2)"}]} + _require(analyze(only_placeholder) == "", "Placeholder-only markdown must not count as a description.") + + with_markdown = copy.deepcopy(CU_IMAGE_SEARCH_RESULT) + with_markdown["contents"][0]["markdown"] = "EXIT 12\n\n![image](pages/1)" + text = analyze(with_markdown) + _require(text == f"EXIT 12\n\n![image](pages/1)\n\n{CU_IMAGE_SUMMARY}", f"Real markdown must be kept, got {text!r}.") + + with_figures = { + "contents": [{ + "kind": "document", + "markdown": "![image](pages/1)", + "fields": {"Summary": {"type": "string", "valueString": CU_IMAGE_SUMMARY}}, + "figures": [ + {"id": "fig-1", "description": "A checkerboard calibration target.", "span": {"offset": 0, "length": 5}}, + {"id": "fig-2", "description": CU_IMAGE_SUMMARY, "span": {"offset": 0, "length": 5}}, + ], + }] + } + text = analyze(with_figures) + _require("Figure (fig-1):\nA checkerboard calibration target." in text, "Figure descriptions must still be appended.") + _require(text.count(CU_IMAGE_SUMMARY) == 1, "A figure description already in the summary must not be repeated.") + + _require(analyze({"contents": []}) == "" and analyze({}) == "", "An empty result must return an empty string.") + + +def check_extraction_engine_image_fallback(content, content_understanding, work_dir): + print("Checking the extraction engine image analyzer fallback...") + image_path = write_test_image(work_dir, "engine-image.png") + di_calls = [] + image_analyzer_calls = [] + warnings = [] + di_pages = [{"page_number": 1, "content": ''}] + no_content_reason = "Content Understanding returned no content, so Document Intelligence Layout was used" + + def run(document_pages, image_result, is_image=True, extraction_mode="layout", document_error=None): + di_calls.clear() + image_analyzer_calls.clear() + warnings.clear() + + def fake_document_analyzer(file_path, pages=None, settings=None): + if document_error: + raise document_error + return copy.deepcopy(document_pages) + + def fake_image_analyzer(file_path, settings=None): + image_analyzer_calls.append(file_path) + if isinstance(image_result, Exception): + raise image_result + return image_result + + def fake_azure_di(file_path, extraction_mode="read", pages=None): + di_calls.append(extraction_mode) + return copy.deepcopy(di_pages) + + def capture_log(message, *args, **kwargs): + warnings.append(str(message)) + + with ExitStack() as stack: + stack.enter_context(patch.object(content_understanding, "extract_content_with_content_understanding", fake_document_analyzer)) + stack.enter_context(patch.object(content_understanding, "analyze_image_with_content_understanding", fake_image_analyzer)) + stack.enter_context(patch.object(content, "extract_content_with_azure_di", fake_azure_di)) + stack.enter_context(patch.object(content, "log_event", capture_log)) + return content.extract_content_with_extraction_engine( + image_path, + extraction_mode=extraction_mode, + extraction_engine="content_understanding", + settings=dict(CU_SETTINGS), + is_image=is_image, + ) + + pages, engine, reason = run([], CU_IMAGE_SUMMARY) + _require(pages == [{"page_number": 1, "content": CU_IMAGE_SUMMARY}], f"The description must become page 1, got {pages}.") + _require(engine == "content_understanding", "The image analyzer result must be attributed to Content Understanding.") + _require(reason == content.CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON and len(reason) < 120, + "The persisted reason must name the image analyzer and stay short.") + _require(not di_calls, "Document Intelligence must not run when the image analyzer described the image.") + + for label, image_result in (("analyzer failure", RuntimeError("image analyzer unavailable")), ("empty description", " ")): + pages, engine, reason = run([], image_result) + _require(pages == di_pages and engine == "document_intelligence" and reason == no_content_reason, + f"{label}: must fall back to Document Intelligence Layout exactly as before.") + _require(di_calls == ["layout"], f"{label}: the fallback must use Layout.") + _require(any("image analyzer" in warning for warning in warnings), f"{label}: a warning must be logged.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, is_image=False) + _require(not image_analyzer_calls and di_calls == ["layout"] and reason == no_content_reason, + "PDFs and other non-image files must keep their existing fallback.") + + cu_pages = [{"page_number": 1, "content": "# Floor plan\n\nRoom 101"}] + pages, engine, reason = run(cu_pages, CU_IMAGE_SUMMARY) + _require(pages == cu_pages and engine == "content_understanding" and reason == "" and not image_analyzer_calls, + "Document analyzer content must be used as-is without calling the image analyzer.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, document_error=RuntimeError("service unavailable")) + _require(not image_analyzer_calls and engine == "document_intelligence" and reason.startswith("Content Understanding failed"), + "A document analyzer failure must keep the existing failure fallback.") + + pages, engine, reason = run([], CU_IMAGE_SUMMARY, extraction_mode="read") + _require(not image_analyzer_calls and di_calls == ["read"] and engine == "document_intelligence", + "Standard extraction must stay on Document Intelligence Read.") + + +def _analyze_with_vision(documents, image_path, settings): + client_builds = [] + legacy_clients = [] + key_vault_lookups = [] + logs = [] + + def fake_key_vault(endpoint_cfg, scope_value, scope="global", return_type=None): + key_vault_lookups.append({"scope_value": scope_value, "scope": scope, "return_type": return_type}) + return endpoint_cfg + + def capture_log(message, extra=None, level=None, **kwargs): + logs.append({"message": str(message), "extra": extra or {}, "level": level}) + + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "build_model_endpoint_sync_chat_client", make_recording_client_builder(client_builds))) + stack.enter_context(patch.object(documents, "keyvault_model_endpoint_get_helper", fake_key_vault)) + stack.enter_context(patch.object(documents, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + stack.enter_context(patch.object(documents, "log_event", capture_log)) + analysis = documents.analyze_image_with_vision_model(image_path, "user-1", "doc-1583", settings) + return analysis, client_builds, legacy_clients, key_vault_lookups, logs + + +def _vision_settings(**overrides): + settings = { + "enable_multimodal_vision": True, + "multimodal_vision_model": "gpt-5.6-luna", + "enable_multi_model_endpoints": True, + "model_endpoints": build_model_endpoints(), + "enable_gpt_apim": False, + "azure_openai_gpt_endpoint": "https://simplechat-openai.invalid", + "azure_openai_gpt_api_version": "2024-12-01-preview", + "azure_openai_gpt_key": "legacy-gpt-key", + "azure_openai_gpt_authentication_type": "key", + } + settings.update(overrides) + return settings + + +def check_vision_model_routes_through_its_connection(documents, work_dir): + print("Checking the vision model is routed through the AI connection that hosts it...") + image_path = write_test_image(work_dir, "vision-image.png") + + endpoint_cfg, model_cfg = documents.resolve_vision_model_endpoint(_vision_settings(), "gpt-5.6-luna") + _require(endpoint_cfg is not None and model_cfg is not None + and endpoint_cfg["id"] == "global-endpoint" and model_cfg["id"] == "luna-model", + "The stored model name must resolve to the Global connection.") + _require(documents.resolve_vision_model_endpoint(_vision_settings(enable_multi_model_endpoints=False), "gpt-5.6-luna") == (None, None), + "Without multi-endpoint models the legacy connection must be used.") + _require(documents.resolve_vision_model_endpoint(_vision_settings(), "") == (None, None), "A blank name must not match.") + + for stored_name in ("gpt-5.6-luna", "luna-prod"): + analysis, builds, legacy_clients, key_vault_lookups, logs = _analyze_with_vision( + documents, image_path, _vision_settings(multimodal_vision_model=stored_name) + ) + _require(analysis and analysis.get("description") == VISION_ANALYSIS["description"], + f"{stored_name}: the routed call must return the parsed analysis.") + _require(analysis.get("model") == stored_name, f"{stored_name}: the analysis must record the configured model name.") + _require(len(builds) == 1 and not legacy_clients, f"{stored_name}: only the Global connection client may be built.") + build_args = builds[0]["args"] + _require(build_args[1:4] == ("aoai", "https://aoai-global-team.invalid", "2025-04-01-preview"), + f"{stored_name}: wrong connection details {build_args[1:4]}.") + _require(build_args[0].get("api_key") == GLOBAL_ENDPOINT_SECRET, "The connection's own credentials must be used.") + _require(builds[0]["kwargs"].get("deployment_name") == "luna-prod", "The client must target the deployment.") + _require(builds[0]["kwargs"].get("identity_context") == {"user_id": "user-1"}, "The caller identity must be passed.") + requests = builds[0]["client"].requests + _require(len(requests) == 1 and requests[0]["model"] == "luna-prod", + f"{stored_name}: the request must send the resolved request model.") + _require(key_vault_lookups and key_vault_lookups[0]["scope_value"] == "global-endpoint", + "Key Vault-backed secrets must be resolved for the matched connection.") + routing_logs = [log for log in logs if "Using the AI connection" in log["message"]] + _require(routing_logs and routing_logs[0]["extra"].get("endpoint_id") == "global-endpoint" + and routing_logs[0]["extra"].get("endpoint_name") == "Global", + "The connection used must be logged by name and id.") + _require(GLOBAL_ENDPOINT_SECRET not in json.dumps(logs, default=str), "Connection secrets must never be logged.") + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + for label, settings in ( + ("multi-endpoint models disabled", _vision_settings(enable_multi_model_endpoints=False)), + ("no matching model", _vision_settings(multimodal_vision_model="gpt-4.1-vision")), + ("matching endpoint disabled", _vision_settings(model_endpoints=disabled_endpoint)), + ("matching model disabled", _vision_settings(model_endpoints=disabled_model)), + ): + analysis, builds, legacy_clients, _, logs = _analyze_with_vision(documents, image_path, settings) + _require(not builds and len(legacy_clients) == 1, f"{label}: the legacy GPT connection must be used.") + legacy = legacy_clients[0] + _require(legacy.init_kwargs == { + "api_version": "2024-12-01-preview", + "azure_endpoint": "https://simplechat-openai.invalid", + "api_key": "legacy-gpt-key", + }, f"{label}: the legacy client must be built exactly as before: {legacy.init_kwargs}.") + _require(legacy.requests[0]["model"] == settings["multimodal_vision_model"], + f"{label}: the legacy request must send the stored model name.") + _require(analysis and analysis.get("description") == VISION_ANALYSIS["description"], f"{label}: analysis expected.") + _require(any("legacy GPT connection" in log["message"] for log in logs), f"{label}: the legacy connection must be logged.") + + analysis, builds, legacy_clients, _, _ = _analyze_with_vision( + documents, + image_path, + _vision_settings( + enable_multi_model_endpoints=False, + enable_gpt_apim=True, + azure_apim_gpt_endpoint="https://apim.invalid", + azure_apim_gpt_api_version="2024-10-21", + azure_apim_gpt_subscription_key="apim-subscription-key", + ), + ) + _require(len(legacy_clients) == 1 and legacy_clients[0].init_kwargs == { + "api_version": "2024-10-21", + "azure_endpoint": "https://apim.invalid", + "api_key": "apim-subscription-key", + }, "The legacy APIM client must be built exactly as before.") + + incomplete = build_model_endpoints() + incomplete[1]["connection"]["endpoint"] = "" + analysis, builds, legacy_clients, _, logs = _analyze_with_vision(documents, image_path, _vision_settings(model_endpoints=incomplete)) + _require(analysis is None and not builds and not legacy_clients, "Errors on the matched connection must return None.") + _require(any("Vision analysis failed" in log["message"] for log in logs), "The failure must be logged.") + + +def _resolve_metadata_client(documents, settings): + client_builds = [] + legacy_clients = [] + key_vault_lookups = [] + + def fake_key_vault(endpoint_cfg, scope_value, scope="global", return_type=None): + key_vault_lookups.append({"scope_value": scope_value, "scope": scope, "return_type": return_type}) + return endpoint_cfg + + sentinel_client = object() + with ExitStack() as stack: + stack.enter_context(patch.object(documents, "build_model_endpoint_sync_chat_client", make_recording_client_builder(client_builds, sentinel_client))) + stack.enter_context(patch.object(documents, "keyvault_model_endpoint_get_helper", fake_key_vault)) + stack.enter_context(patch.object(documents, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + try: + result = documents._resolve_metadata_extraction_client(settings, identity_context={"user_id": "user-1"}) + error = None + except Exception as resolution_error: # noqa: BLE001 - the check compares the exact error + result = None + error = resolution_error + return result, error, client_builds, legacy_clients, key_vault_lookups, sentinel_client + + +def check_metadata_extraction_client_is_unchanged(documents): + print("Checking the metadata extraction client is built exactly as before...") + + def settings_with(selection=None, endpoints=None, **overrides): + settings = { + "enable_multi_model_endpoints": True, + "model_endpoints": endpoints if endpoints is not None else build_model_endpoints(), + "metadata_extraction_model_selection": selection or { + "endpoint_id": "global-endpoint", + "model_id": "luna-model", + "provider": "aoai", + }, + "metadata_extraction_model": "gpt-4o", + "enable_gpt_apim": False, + "azure_openai_gpt_endpoint": "https://simplechat-openai.invalid", + "azure_openai_gpt_api_version": "2024-12-01-preview", + "azure_openai_gpt_key": "legacy-gpt-key", + "azure_openai_gpt_authentication_type": "key", + } + settings.update(overrides) + return settings + + settings = settings_with() + result, error, builds, legacy_clients, key_vault_lookups, sentinel = _resolve_metadata_client(documents, settings) + _require(error is None, f"The selected metadata model must resolve, got {error!r}.") + _require(result == (sentinel, "luna-prod"), f"Expected (client, deployment), got {result!r}.") + _require(not legacy_clients and len(builds) == 1, "Only the selected connection client may be built.") + args, kwargs = builds[0]["args"], builds[0]["kwargs"] + _require(args[1:4] == ("aoai", "https://aoai-global-team.invalid", "2025-04-01-preview"), + f"Unexpected connection details: {args[1:4]}.") + _require(args[0].get("type") == "api_key" and args[0].get("api_key") == GLOBAL_ENDPOINT_SECRET, + "The endpoint auth must be passed through.") + _require(kwargs.get("deployment_name") == "luna-prod" and kwargs.get("api_type") == "" and kwargs.get("anthropic_version") == "", + f"Unexpected client arguments: {kwargs}.") + _require(kwargs.get("allow_private_custom_endpoints") is False and kwargs.get("settings") is settings, + "The private endpoint flag and settings must be passed through.") + _require(kwargs.get("endpoint_config", {}).get("id") == "global-endpoint", "The resolved endpoint config must be passed.") + _require(kwargs.get("identity_context") == {"user_id": "user-1"}, "The identity context must be passed through.") + _require(key_vault_lookups == [{"scope_value": "global-endpoint", "scope": "global", "return_type": documents.SecretReturnType.VALUE}], + f"Key Vault secrets must be resolved exactly once for the endpoint: {key_vault_lookups}.") + + disabled_endpoint = build_model_endpoints() + disabled_endpoint[1]["enabled"] = False + disabled_model = build_model_endpoints() + disabled_model[1]["models"][0]["enabled"] = False + unsupported = build_model_endpoints() + unsupported[1]["provider"] = "Bogus" + incomplete = build_model_endpoints() + incomplete[1]["connection"]["endpoint"] = "" + no_api_version = build_model_endpoints() + no_api_version[1]["connection"].pop("openai_api_version") + + for label, settings, expected_type, expected_message, expects_key_vault in ( + ("missing endpoint", settings_with({"endpoint_id": "missing", "model_id": "luna-model"}), + LookupError, "Selected metadata extraction endpoint could not be found.", False), + ("disabled endpoint", settings_with(endpoints=disabled_endpoint), + ValueError, "Selected metadata extraction endpoint is disabled.", False), + ("missing model", settings_with({"endpoint_id": "global-endpoint", "model_id": "missing"}), + LookupError, "Selected metadata extraction model could not be found on the endpoint.", True), + ("disabled model", settings_with(endpoints=disabled_model), + ValueError, "Selected metadata extraction model is disabled.", True), + ("unsupported provider", settings_with(endpoints=unsupported), + ValueError, "Selected metadata extraction provider 'bogus' is not supported.", True), + ("incomplete endpoint", settings_with(endpoints=incomplete), + ValueError, "Selected metadata extraction endpoint is incomplete.", True), + ("missing API version", settings_with(endpoints=no_api_version), + ValueError, "Selected metadata extraction endpoint is incomplete.", True), + ): + result, error, builds, _, key_vault_lookups, _ = _resolve_metadata_client(documents, settings) + _require(type(error) is expected_type and str(error) == expected_message, + f"{label}: expected {expected_type.__name__}({expected_message!r}), got {error!r}.") + _require(not builds, f"{label}: no client may be built.") + _require(bool(key_vault_lookups) == expects_key_vault, f"{label}: Key Vault lookup order changed.") + + result, error, builds, legacy_clients, _, _ = _resolve_metadata_client( + documents, settings_with(enable_multi_model_endpoints=False) + ) + _require(error is None and not builds and len(legacy_clients) == 1 and result == (legacy_clients[0], "gpt-4o"), + "Without multi-endpoint models the legacy metadata extraction client must be used.") + _require(legacy_clients[0].init_kwargs.get("azure_endpoint") == "https://simplechat-openai.invalid" + and legacy_clients[0].init_kwargs.get("api_key") == "legacy-gpt-key", + "The legacy metadata extraction client must be built from the GPT settings.") + + result, error, _, _, _, _ = _resolve_metadata_client( + documents, settings_with(enable_multi_model_endpoints=False, metadata_extraction_model="") + ) + _require(type(error) is ValueError and str(error) == "No metadata extraction model is selected.", + "A missing legacy metadata model must keep its error.") + + +def check_vision_test_uses_the_ingestion_connection(route_settings, keyvault): + print("Checking the admin vision test uses the same connection as ingestion...") + from flask import Flask # noqa: PLC0415 - imported inside the offline bootstrap + + web = Flask("image-description-vision-test") + + def run(saved_settings, payload): + client_builds = [] + legacy_clients = [] + with ExitStack() as stack: + stack.enter_context(patch.object(route_settings, "get_settings", lambda *a, **k: copy.deepcopy(saved_settings))) + stack.enter_context(patch.object(route_settings, "get_current_user_id", lambda: "admin-user")) + stack.enter_context(patch.object(route_settings, "build_model_endpoint_sync_chat_client", + make_recording_client_builder(client_builds))) + stack.enter_context(patch.object(route_settings, "AzureOpenAI", make_recording_azure_openai(legacy_clients))) + stack.enter_context(patch.object(keyvault, "keyvault_model_endpoint_get_helper", + lambda endpoint_cfg, *a, **k: endpoint_cfg)) + with web.app_context(): + response, status_code = route_settings._test_multimodal_vision_connection(copy.deepcopy(payload)) + return response.get_json(), status_code, client_builds, legacy_clients + + apim_payload = { + "test_type": "multimodal_vision", + "enable_apim": True, + "vision_model": "gpt-5.6-luna", + "apim": { + "endpoint": "https://apim.invalid", + "subscription_key": "apim-subscription-key", + "api_version": "2024-10-21", + "deployment": "gpt-5.6-luna", + }, + } + saved_settings = _vision_settings(enable_gpt_apim=True) + + body, status_code, builds, legacy_clients = run(saved_settings, apim_payload) + _require(status_code == 200, f"The vision test must succeed, got {status_code}: {body}.") + _require(len(builds) == 1 and not legacy_clients, + "With the model hosted on an AI connection, the test must not use APIM, matching ingestion.") + _require(builds[0]["args"][2] == "https://aoai-global-team.invalid" and builds[0]["kwargs"].get("deployment_name") == "luna-prod", + "The test must call the same connection and deployment that ingestion uses.") + + direct_payload = { + "test_type": "multimodal_vision", + "enable_apim": False, + "vision_model": "gpt-5.6-luna", + "multi_endpoint": { + "endpoint_id": "global-endpoint", + "model_id": "luna-model", + "provider": "aoai", + "model_name": "gpt-5.6-luna", + "deployment_name": "gpt-5.6-luna", + }, + "direct": {"endpoint": "https://simplechat-openai.invalid", "auth_type": "key", "key": "legacy-gpt-key", + "api_version": "2024-12-01-preview", "deployment": "gpt-5.6-luna"}, + } + body, status_code, builds, legacy_clients = run(_vision_settings(), direct_payload) + _require(status_code == 200 and len(builds) == 1 and not legacy_clients + and builds[0]["args"][2] == "https://aoai-global-team.invalid", + "The classic multi-endpoint payload must keep resolving the Global connection.") + + body, status_code, builds, legacy_clients = run(_vision_settings(enable_multi_model_endpoints=False, enable_gpt_apim=True), apim_payload) + _require(status_code == 200 and not builds and len(legacy_clients) == 1 + and legacy_clients[0].init_kwargs.get("azure_endpoint") == "https://apim.invalid", + "Without multi-endpoint models the APIM test path must be unchanged.") + + +def _run_offline_probe(): + # Real application imports must occur inside the external-I/O bootstrap seam. + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline, tempfile.TemporaryDirectory() as work_dir: + _require("functions_content" not in sys.modules, "functions_content was imported before the probe.") + import functions_content as content # noqa: PLC0415 + import functions_content_understanding as content_understanding # noqa: PLC0415 + import functions_documents as documents # noqa: PLC0415 + import functions_keyvault as keyvault # noqa: PLC0415 + import route_backend_settings as route_settings # noqa: PLC0415 + _require(not offline.network_attempts, "Application imports attempted network access.") + + check_vision_block_formatting(documents) + check_comment_only_ocr_is_not_text(documents) + check_save_chunks_embeds_the_stored_text(documents) + check_save_chunks_batch_matches_save_chunks(documents, content) + check_text_free_images_are_indexed_from_vision(documents, work_dir) + check_content_understanding_image_analyzer(content_understanding, work_dir) + check_extraction_engine_image_fallback(content, content_understanding, work_dir) + check_vision_model_routes_through_its_connection(documents, work_dir) + check_metadata_extraction_client_is_unchanged(documents) + check_vision_test_uses_the_ingestion_connection(route_settings, keyvault) + _require(not offline.network_attempts, "The checks attempted network access.") + + print("All offline image description indexing checks passed.") + + +def _run_reverse_import_order_probe(): + """Import Content Understanding before functions_content, the reverse of the main probe. + + extract_content_with_extraction_engine() imports functions_content_understanding when it + runs, and functions_content_understanding reaches functions_content back through + functions_settings, functions_document_actions, and functions_search. The image analyzer + fallback must resolve both deferred names whichever module is imported first. + """ + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline, tempfile.TemporaryDirectory() as work_dir: + for module_name in ("functions_content", "functions_content_understanding"): + _require(module_name not in sys.modules, f"{module_name} was imported before the probe.") + import functions_content_understanding as content_understanding # noqa: PLC0415 + _require("functions_content" in sys.modules, + "Importing functions_content_understanding no longer reaches functions_content; update this probe.") + import functions_content as content # noqa: PLC0415 + _require(not offline.network_attempts, "Application imports attempted network access.") + + check_extraction_engine_image_fallback(content, content_understanding, work_dir) + _require(not offline.network_attempts, "The checks attempted network access.") + + print("Reverse import order checks passed.") + + +def _run_probe_process(probe_flag, optimized=False): + """Run a probe in a fresh interpreter so imports start from a clean module table.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(APP_ROOT), str(TEST_ROOT))) + env["PYTHONIOENCODING"] = "utf-8" + command = [sys.executable] + if optimized: + command.append("-O") + command.extend((str(Path(__file__).resolve()), probe_flag)) + return subprocess.run( + command, cwd=ROOT, env=env, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=300, + ) + + +@pytest.mark.parametrize("optimized", (False, True)) +def test_image_description_indexing_real_modules(optimized): + """Run the real-module checks in a fresh process with external I/O blocked.""" + result = _run_probe_process("--offline-probe", optimized=optimized) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_image_analyzer_fallback_with_reverse_import_order(): + """The image analyzer fallback must work when Content Understanding is imported first.""" + result = _run_probe_process("--reverse-import-order-probe") + assert result.returncode == 0, result.stdout + result.stderr + + +if __name__ == "__main__": + if "--offline-probe" in sys.argv: + _run_offline_probe() + elif "--reverse-import-order-probe" in sys.argv: + _run_reverse_import_order_probe() + else: + raise SystemExit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_image_generation_api_version_default.py b/functional_tests/test_image_generation_api_version_default.py new file mode 100644 index 000000000..62be0b52f --- /dev/null +++ b/functional_tests/test_image_generation_api_version_default.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# test_image_generation_api_version_default.py +""" +Functional test for the Azure OpenAI image generation API version default and its migration. +Version: 0.261.047 +Implemented in: 0.261.047 + +azure_openai_image_gen_api_version defaulted to 2024-12-01-preview, which predates gpt-image +model support. The default is now 2025-04-01-preview, the newest dated Azure OpenAI API version and +the one Microsoft documents for gpt-image-1, gpt-image-1.5, and gpt-image-2. deep_merge_dicts() +never overwrites a key that a persisted settings document already has, so existing deployments +keep the old default unless it is migrated when settings load. + +This test ensures that: + - the shipped default is 2025-04-01-preview, + - normalize_image_generation_api_version_settings() upgrades the previous default and blank + values, leaves any other value untouched, and never changes the APIM API version, + - the migration runs on every settings load and is persisted to the settings document, + - the admin documentation and example settings artifacts show the new default. + +The migration helper is loaded from source without importing functions_settings, which builds +Azure clients at import time. The end-to-end load check imports the real module in fresh normal +and optimized processes with external I/O blocked. +""" + +import ast +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" +TEST_ROOT = ROOT / "functional_tests" +SETTINGS_FILE = APP_ROOT / "functions_settings.py" +IMPLEMENTED_IN_VERSION = "0.261.047" +NEW_DEFAULT = "2025-04-01-preview" +PREVIOUS_DEFAULT = "2024-12-01-preview" +MIGRATION_FUNCTION = "normalize_image_generation_api_version_settings" +DEFAULT_CONSTANT = "AZURE_OPENAI_IMAGE_GEN_API_VERSION_DEFAULT" +PREVIOUS_DEFAULT_CONSTANT = "AZURE_OPENAI_IMAGE_GEN_API_VERSION_PREVIOUS_DEFAULT" + +if str(TEST_ROOT) not in sys.path: + sys.path.insert(0, str(TEST_ROOT)) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +def _require(condition, message): + """Raise explicitly so checks survive python -O.""" + if not condition: + raise AssertionError(message) + + +def load_migration(): + """Load the migration helper and its constants from source, without importing Azure config.""" + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + wanted_constants = {DEFAULT_CONSTANT, PREVIOUS_DEFAULT_CONSTANT} + + selected_nodes = [] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == MIGRATION_FUNCTION: + selected_nodes.append(node) + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in wanted_constants + ): + selected_nodes.append(node) + + assert len(selected_nodes) == 3, ( + f"Expected {DEFAULT_CONSTANT}, {PREVIOUS_DEFAULT_CONSTANT}, and {MIGRATION_FUNCTION}() " + "at module level in functions_settings.py" + ) + + namespace = {} + exec(compile(ast.Module(body=selected_nodes, type_ignores=[]), str(SETTINGS_FILE), "exec"), namespace) + return namespace + + +def _function_node(tree, name): + return next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == name) + + +def test_default_is_the_newest_gpt_image_api_version(): + """The code-level default must be 2025-04-01-preview.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + namespace = load_migration() + assert namespace[DEFAULT_CONSTANT] == NEW_DEFAULT + assert namespace[PREVIOUS_DEFAULT_CONSTANT] == PREVIOUS_DEFAULT + + source = SETTINGS_FILE.read_text(encoding="utf-8") + assert f"'azure_openai_image_gen_api_version': {DEFAULT_CONSTANT}," in source, ( + "get_settings() defaults must use the named image generation API version constant" + ) + assert f"'azure_openai_image_gen_api_version': '{PREVIOUS_DEFAULT}'" not in source + assert "'azure_apim_image_gen_api_version': ''," in source, ( + "The APIM image generation API version must keep having no default" + ) + + +def test_previous_default_is_upgraded(): + """A stored value equal to the previous default was never customized, so it is upgraded.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = { + "azure_openai_image_gen_api_version": PREVIOUS_DEFAULT, + "azure_apim_image_gen_api_version": "", + "unrelated_key": "left alone", + } + + changed = migrate(settings) + + assert changed is True + assert settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert settings["azure_apim_image_gen_api_version"] == "" + assert settings["unrelated_key"] == "left alone" + + padded = {"azure_openai_image_gen_api_version": f" {PREVIOUS_DEFAULT} "} + assert migrate(padded) is True and padded["azure_openai_image_gen_api_version"] == NEW_DEFAULT + + +@pytest.mark.parametrize("stored_version", ["", " ", None]) +def test_blank_values_are_upgraded(stored_version): + """A blank stored value would send no API version, so it becomes the new default.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {"azure_openai_image_gen_api_version": stored_version} + + changed = migrate(settings) + + assert changed is True + assert settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + + +def test_missing_value_is_filled(): + """A document without the key gets the new default rather than nothing.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {} + + changed = migrate(settings) + + assert changed is True + assert settings == {"azure_openai_image_gen_api_version": NEW_DEFAULT} + + +@pytest.mark.parametrize("custom_version", ["2025-03-01-preview", "2024-10-21", "2024-05-01-preview", NEW_DEFAULT]) +def test_custom_values_are_left_untouched(custom_version): + """Any other value is an admin's deliberate choice, and an up-to-date value needs no write.""" + migrate = load_migration()[MIGRATION_FUNCTION] + settings = {"azure_openai_image_gen_api_version": custom_version} + + changed = migrate(settings) + + assert changed is False + assert settings["azure_openai_image_gen_api_version"] == custom_version + + +def test_apim_version_is_never_changed(): + """APIM gateways define their own API version, so it is never migrated.""" + migrate = load_migration()[MIGRATION_FUNCTION] + for apim_version in (PREVIOUS_DEFAULT, "", " ", "2024-02-01"): + settings = { + "azure_openai_image_gen_api_version": PREVIOUS_DEFAULT, + "azure_apim_image_gen_api_version": apim_version, + } + migrate(settings) + assert settings["azure_apim_image_gen_api_version"] == apim_version, ( + f"The APIM API version {apim_version!r} must not be migrated" + ) + + apim_only = {"azure_openai_image_gen_api_version": "2025-03-01-preview", "azure_apim_image_gen_api_version": PREVIOUS_DEFAULT} + assert migrate(apim_only) is False + assert apim_only["azure_apim_image_gen_api_version"] == PREVIOUS_DEFAULT + + +def test_unexpected_input_is_handled_safely(): + """Non-dict settings and non-string values must not raise or be rewritten.""" + migrate = load_migration()[MIGRATION_FUNCTION] + assert migrate(None) is False + assert migrate("not-a-dict") is False + + settings = {"azure_openai_image_gen_api_version": 20250401} + assert migrate(settings) is False + assert settings["azure_openai_image_gen_api_version"] == 20250401 + + +def test_migration_is_wired_into_every_settings_load(): + """get_settings() must run the migration inside normalize_loaded_settings().""" + tree = ast.parse(SETTINGS_FILE.read_text(encoding="utf-8"), filename=str(SETTINGS_FILE)) + loader = _function_node(_function_node(tree, "get_settings"), "normalize_loaded_settings") + calls = [ + node for node in ast.walk(loader) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == MIGRATION_FUNCTION + ] + assert len(calls) == 1, f"normalize_loaded_settings() must call {MIGRATION_FUNCTION}() once" + assert [ast.unparse(arg) for arg in calls[0].args] == ["merged"], "The migration must run on the merged settings" + + +def test_docs_and_example_artifacts_show_the_new_default(): + """Admin docs and example settings must match the shipped image generation default.""" + ai_models_doc = (ROOT / "docs" / "admin" / "ai-models.md").read_text(encoding="utf-8") + image_row = next(line for line in ai_models_doc.splitlines() if "`azure_openai_image_gen_api_version`" in line) + assert f"| {NEW_DEFAULT} |" in image_row, f"The Image Gen API Version row must show {NEW_DEFAULT}: {image_row}" + apim_row = next(line for line in ai_models_doc.splitlines() if "`azure_apim_image_gen_api_version`" in line) + assert "| Empty |" in apim_row, "The APIM image generation API version must still have no default" + + cosmos_example = json.loads((ROOT / "artifacts" / "cosmos_examples" / "cosmos-settings-example.json").read_text(encoding="utf-8")) + assert cosmos_example["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert cosmos_example["azure_openai_gpt_api_version"] == PREVIOUS_DEFAULT, "Only the image generation entry changes" + assert cosmos_example["azure_openai_embedding_api_version"] == PREVIOUS_DEFAULT, "Only the image generation entry changes" + + seeder_settings = json.loads( + (ROOT / "application" / "external_apps" / "databaseseeder" / "artifacts" / "admin_settings.json").read_text(encoding="utf-8") + ) + assert seeder_settings["azure_openai_image_gen_api_version"] == NEW_DEFAULT + assert seeder_settings["azure_apim_image_gen_api_version"] == "" + + +def _run_offline_probe(): + # Real application imports must occur inside the external-I/O bootstrap seam. + from test_support.offline_bootstrap import offline_app_imports # noqa: PLC0415 + + with offline_app_imports() as offline: + import functions_settings as settings_module # noqa: PLC0415 + _require(not offline.network_attempts, "Settings imports attempted network access.") + container = settings_module.cosmos_settings_container + + for stored_version, apim_version, expected_version in ( + (PREVIOUS_DEFAULT, PREVIOUS_DEFAULT, NEW_DEFAULT), + ("", "", NEW_DEFAULT), + (" ", "2024-02-01", NEW_DEFAULT), + ("2025-03-01-preview", "", "2025-03-01-preview"), + ): + container.upsert_item({ + "id": "app_settings", + "azure_openai_image_gen_api_version": stored_version, + "azure_apim_image_gen_api_version": apim_version, + }) + + loaded = settings_module.get_settings() + _require(isinstance(loaded, dict), "Settings must load.") + _require(loaded["azure_openai_image_gen_api_version"] == expected_version, + f"Loaded {loaded['azure_openai_image_gen_api_version']!r} for stored {stored_version!r}.") + _require(loaded["azure_apim_image_gen_api_version"] == apim_version, + f"The APIM version {apim_version!r} must not be migrated.") + + persisted = container.read_item(item="app_settings", partition_key="app_settings") + _require(persisted["azure_openai_image_gen_api_version"] == expected_version, + f"The migrated value must be persisted for stored {stored_version!r}.") + _require(persisted["azure_apim_image_gen_api_version"] == apim_version, + "The persisted APIM version must be unchanged.") + + _require(settings_module.get_settings()["azure_openai_image_gen_api_version"] == "2025-03-01-preview", + "A custom value must survive repeated loads.") + _require(not offline.network_attempts, "Settings loads attempted network access.") + + print("Image generation API version load migration checks passed.") + + +@pytest.mark.parametrize("optimized", (False, True)) +def test_settings_load_migrates_and_persists_the_api_version(optimized): + """Run the real get_settings() load path in a fresh process with external I/O blocked.""" + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(APP_ROOT), str(TEST_ROOT))) + env["PYTHONIOENCODING"] = "utf-8" + command = [sys.executable] + if optimized: + command.append("-O") + command.extend((str(Path(__file__).resolve()), "--offline-probe")) + result = subprocess.run( + command, cwd=ROOT, env=env, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=300, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_version_is_at_least_implementation_version(): + """The app version must be at or beyond the version this fix shipped in.""" + assert_app_version_at_least(IMPLEMENTED_IN_VERSION) + + +if __name__ == "__main__": + if "--offline-probe" in sys.argv: + _run_offline_probe() + else: + raise SystemExit(pytest.main([__file__, "-q"]))