Show what was extracted from V2 image uploads, index text-free images, and default image generation to 2025-04-01-preview

application/external_apps/databaseseeder/artifacts/admin_settings.json

@@ -37,7 +37,7 @@
     "enable_image_generation": false,
     "enable_image_gen_apim": false,
     "azure_openai_image_gen_endpoint": "",
-    "azure_openai_image_gen_api_version": "2024-05-01-preview",
+    "azure_openai_image_gen_api_version": "2025-04-01-preview",
     "azure_openai_image_gen_authentication_type": "key",
     "azure_openai_image_gen_subscription_id": "",
     "azure_openai_image_gen_resource_group": "",

application/single_app/admin_settings_fields.py

@@ -2320,8 +2320,8 @@
             "type": "switch",
             "label": "Enable Multi-Modal Vision Analysis",
             "help": (
-                "Sends page images to a vision-capable model so figures, charts and "
-                "scanned pages are described and indexed rather than skipped."
+                "Asks a vision-capable model to describe each uploaded image, so the image "
+                "is indexed and can be cited by what it shows even when it contains no text."
             ),
             "default": False,
             "role": "capability",
@@ -5909,7 +5909,7 @@
                 "only; a chat deployment reached through the Responses image tool uses a "
                 "version new enough for that route regardless of what is set here."
             ),
-            "default": "2024-12-01-preview",
+            "default": "2025-04-01-preview",
             "depends_on": [
                 {"key": "enable_image_generation", "equals": True},
                 {"key": "enable_image_gen_apim", "equals": False},

application/single_app/config.py

@@ -100,7 +100,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.209"
+VERSION = "0.261.210"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/functions_chat_upload_extraction.py

@@ -0,0 +1,166 @@
+# functions_chat_upload_extraction.py
+"""Describe what ingestion extracted from the workspace document behind a chat upload.
+
+A file uploaded into a conversation is stored as a workspace document, so the chat message
+itself carries no text. Opening the upload therefore has to read what ingestion produced for
+that document: the indexed text that search and chat actually use, and the structured image
+analysis that was saved on the document record but was never shown anywhere.
+
+Everything here is pure so the route can stay thin and the shaping can be tested without
+Cosmos DB, AI Search, or Azure credentials.
+"""
+
+WORKSPACE_UPLOAD_FILE_CONTENT_SOURCE = "workspace"
+NO_EXTRACTED_CONTENT_MESSAGE = "No extracted content is available for this file yet."
+STILL_PROCESSING_MESSAGE = "This file is still being processed. Try again when it finishes."
+PROCESSING_FAILED_MESSAGE = (
+    "Processing this file failed, so nothing was extracted. Reprocess or re-upload it to try again."
+)
+
+
+def _clean_text(value):
+    return value.strip() if isinstance(value, str) else ""
+
+
+def _clean_list(values):
+    if isinstance(values, str):
+        values = [values]
+    if not isinstance(values, (list, tuple)):
+        return []
+    return [value.strip() for value in values if isinstance(value, str) and value.strip()]
+
+
+def summarize_vision_analysis(vision_analysis):
+    """Return the displayable parts of a stored vision analysis, or None when it has none.
+
+    An analysis that records an error is excluded, because its description is the error
+    message rather than anything the model saw in the image.
+    """
+    if not isinstance(vision_analysis, dict) or vision_analysis.get("error"):
+        return None
+
+    summary = {
+        "model": _clean_text(vision_analysis.get("model") or vision_analysis.get("model_name")),
+        "description": _clean_text(vision_analysis.get("description")),
+        "objects": _clean_list(vision_analysis.get("objects")),
+        "text": _clean_text(vision_analysis.get("text")),
+        "analysis": _clean_text(
+            vision_analysis.get("analysis") or vision_analysis.get("contextual_analysis")
+        ),
+    }
+    if not any(summary[key] for key in ("description", "objects", "text", "analysis")):
+        return None
+    return summary
+
+
+def render_vision_analysis_text(summary):
+    """Plain-text rendering for clients that can only show a text body."""
+    if not summary:
+        return ""
+
+    lines = ["AI Vision Analysis"]
+    if summary.get("model"):
+        lines.append(f"Model: {summary['model']}")
+    if summary.get("description"):
+        lines.append(f"\nDescription: {summary['description']}")
+    if summary.get("objects"):
+        lines.append(f"\nObjects Detected: {', '.join(summary['objects'])}")
+    if summary.get("text"):
+        lines.append(f"\nVisible Text: {summary['text']}")
+    if summary.get("analysis"):
+        lines.append(f"\nContextual Analysis: {summary['analysis']}")
+    return "\n".join(lines).strip()
+
+
+def join_indexed_chunk_text(chunks):
+    """Join the indexed text of ordered chunks, skipping empty ones."""
+    parts = []
+    for chunk in chunks or []:
+        if not isinstance(chunk, dict):
+            continue
+        text = _clean_text(chunk.get("chunk_text"))
+        if text:
+            parts.append(text)
+    return "\n\n".join(parts)
+
+
+def _percentage_complete(document):
+    try:
+        return int(document.get("percentage_complete"))
+    except (TypeError, ValueError):
+        return None
+
+
+def is_workspace_upload_failed(document):
+    """Return whether ingestion stopped with an error, which is final rather than in progress."""
+    return _clean_text(document.get("status")).lower().startswith("error")
+
+
+def is_workspace_upload_processing(document):
+    """Return whether the linked document has not finished ingestion yet."""
+    if is_workspace_upload_failed(document):
+        return False
+    percentage = _percentage_complete(document)
+    if percentage is not None:
+        return percentage < 100
+    status = _clean_text(document.get("status")).lower()
+    return bool(status) and not status.startswith("processing complete")
+
+
+def build_workspace_upload_details(document, indexed_chunk_count):
+    """Return the browser-safe extraction details of the document behind a chat upload.
+
+    Only descriptive ingestion results are returned. Storage locations, scope identifiers and
+    anything the screening subsystem manages stay on the server.
+    """
+    source = document if isinstance(document, dict) else {}
+    try:
+        number_of_pages = int(source.get("number_of_pages"))
+    except (TypeError, ValueError):
+        number_of_pages = None
+
+    return {
+        "document_id": _clean_text(source.get("id")),
+        "title": _clean_text(source.get("title")),
+        "abstract": _clean_text(source.get("abstract")),
+        "keywords": _clean_list(source.get("keywords")),
+        "status": _clean_text(source.get("status")),
+        "percentage_complete": _percentage_complete(source),
+        "number_of_pages": number_of_pages,
+        "extraction_engine": _clean_text(source.get("extraction_engine")),
+        "extraction_engine_reason": _clean_text(source.get("extraction_engine_reason")),
+        "indexed_chunk_count": max(0, int(indexed_chunk_count or 0)),
+        "vision_analysis": summarize_vision_analysis(source.get("vision_analysis")),
+    }
+
+
+def build_workspace_upload_file_content(filename, document, chunks):
+    """Shape the file-content response for a chat upload backed by a workspace document.
+
+    Returns ``(payload, status_code)``. ``file_content`` is always a readable body when the
+    response succeeds, so a client that only renders text still has something to show: the
+    indexed text when ingestion saved any, otherwise the stored image analysis.
+    """
+    ordered_chunks = [chunk for chunk in chunks or [] if isinstance(chunk, dict)]
+    indexed_text = join_indexed_chunk_text(ordered_chunks)
+    details = build_workspace_upload_details(document, len(ordered_chunks))
+    body = indexed_text or render_vision_analysis_text(details["vision_analysis"])
+
+    if not body:
+        source = document if isinstance(document, dict) else {}
+        if is_workspace_upload_failed(source):
+            message = PROCESSING_FAILED_MESSAGE
+        elif is_workspace_upload_processing(source):
+            message = STILL_PROCESSING_MESSAGE
+        else:
+            message = NO_EXTRACTED_CONTENT_MESSAGE
+        return {"error": message, "workspace_document": details}, 404
+
+    return {
+        "file_content": body,
+        "filename": filename,
+        "is_table": False,
+        "file_content_source": WORKSPACE_UPLOAD_FILE_CONTENT_SOURCE,
+        "indexed_text_available": bool(indexed_text),
+        "workspace_document": details,
+    }, 200

application/single_app/functions_content.py

@@ -625,19 +625,53 @@ def resolve_extraction_engine_for_mode(extraction_mode, settings):
     return functions_settings.resolve_enhanced_extraction_engine(settings)
 
 
+# Persisted as the extraction engine reason and shown in workspace tooltips, so keep it short.
+CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON = (
+    'Content Understanding found no text, so its image analyzer description was indexed'
+)
+
+
+def _describe_image_with_content_understanding(file_path, settings, analyze_image):
+    """Return the Content Understanding image analyzer description, or '' when it has none."""
+    try:
+        description_text = str(analyze_image(file_path, settings=settings) or '').strip()
+    except Exception as image_analyzer_error:
+        log_event(
+            f"[EXTRACTION_ENGINE] Content Understanding image analyzer failed for "
+            f"{os.path.basename(file_path)}: {image_analyzer_error}. "
+            "Falling back to Document Intelligence Layout.",
+            level=logging.WARNING,
+        )
+        return ''
+
+    if not description_text:
+        log_event(
+            f"[EXTRACTION_ENGINE] Content Understanding image analyzer returned no description for "
+            f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.",
+            level=logging.WARNING,
+        )
+    return description_text
+
+
 def extract_content_with_extraction_engine(
     file_path,
     extraction_mode,
     extraction_engine=None,
     settings=None,
     pages=None,
+    is_image=False,
 ):
     """Extract page content with the engine that backs the requested extraction mode.
 
     Standard extraction always uses Document Intelligence ``prebuilt-read``. Enhanced extraction
     prefers Azure AI Content Understanding, but always falls back to Document Intelligence
     ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion.
 
+    When ``is_image`` is set and the document analyzer finds no content, which is normal for a
+    photo with no text, the Content Understanding image analyzer's description is returned as
+    the image's single page instead. Only if that also yields nothing does extraction fall back
+    to Document Intelligence Layout.
+
     Returns ``(pages_data, engine_used, fallback_reason)``.
     """
     resolved_settings = settings if settings is not None else get_settings()
@@ -654,7 +688,10 @@ def extract_content_with_extraction_engine(
             '',
         )
 
-    from functions_content_understanding import extract_content_with_content_understanding
+    from functions_content_understanding import (
+        analyze_image_with_content_understanding,
+        extract_content_with_content_understanding,
+    )
 
     try:
         content_understanding_pages = extract_content_with_content_understanding(
@@ -668,6 +705,18 @@ def extract_content_with_extraction_engine(
                 functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING,
                 '',
             )
+        if is_image:
+            image_description = _describe_image_with_content_understanding(
+                file_path,
+                resolved_settings,
+                analyze_image_with_content_understanding,
+            )
+            if image_description:
+                return (
+                    [{'page_number': 1, 'content': image_description}],
+                    functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING,
+                    CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON,
+                )
         fallback_reason = (
             'Content Understanding returned no content, so Document Intelligence Layout was used'
         )

application/single_app/functions_content_understanding.py

@@ -14,6 +14,7 @@
 import json
 import logging
 import os
+import re
 import time
 
 import requests
@@ -48,6 +49,11 @@
     "Understanding defaults before using Enhanced extraction."
 )
 
+# Image analyzers such as prebuilt-imageSearch return a bare image reference, for example
+# "![image](pages/1)", as the markdown and put the actual description in a Summary field.
+MARKDOWN_IMAGE_PLACEHOLDER_PATTERN = re.compile(r'!\[[^\]]*\]\([^)]*\)')
+CONTENT_UNDERSTANDING_SUMMARY_FIELD = 'summary'
+
 
 class ContentUnderstandingError(Exception):
     """Raised when a Content Understanding request cannot be completed."""
@@ -521,8 +527,45 @@ def extract_content_with_content_understanding(file_path, pages=None, settings=N
     return pages_data
 
 
+def _is_image_placeholder_markdown(markdown_text):
+    """Return True when markdown holds nothing but image references such as ``![image](pages/1)``."""
+    return not MARKDOWN_IMAGE_PLACEHOLDER_PATTERN.sub('', str(markdown_text or '')).strip()
+
+
+def _build_string_field_blocks(content):
+    """Return readable text for the non-empty string fields an analyzer extracted.
+
+    The ``Summary`` field is emitted as plain text first, because it is the analyzer's own
+    description of the input. Every other string field is emitted as ``Name: value``.
+    """
+    fields = content.get('fields')
+    if not isinstance(fields, dict):
+        return []
+
+    summary_blocks = []
+    field_blocks = []
+    for field_name, field_value in fields.items():
+        if not isinstance(field_value, dict):
+            continue
+        value_string = field_value.get('valueString')
+        if not isinstance(value_string, str) or not value_string.strip():
+            continue
+
+        if str(field_name).strip().lower() == CONTENT_UNDERSTANDING_SUMMARY_FIELD:
+            summary_blocks.append(value_string.strip())
+        else:
+            field_blocks.append(f"{field_name}: {value_string.strip()}")
+
+    return summary_blocks + field_blocks
+
+
 def analyze_image_with_content_understanding(image_path, settings=None):
-    """Analyze a standalone image and return its description text, or an empty string."""
+    """Analyze a standalone image and return its description text, or an empty string.
+
+    Image analyzers describe the image in extracted fields rather than in the markdown, which is
+    only a placeholder image reference, so string fields are read alongside any real markdown
+    and figure descriptions.
+    """
     resolved_settings = settings if settings is not None else _get_settings()
     config = _resolve_config(settings=resolved_settings)
 
@@ -535,15 +578,27 @@ def analyze_image_with_content_understanding(image_path, settings=None):
     text_blocks = []
     for content in _iter_document_contents(result):
         markdown_text = str(content.get('markdown') or '').strip()
+        if _is_image_placeholder_markdown(markdown_text):
+            markdown_text = ''
         if markdown_text:
             text_blocks.append(markdown_text)
 
+        field_blocks = _build_string_field_blocks(content)
+        text_blocks.extend(field_blocks)
+
+        # De-duplicate figure descriptions against everything already emitted for this content.
+        emitted_text = "\n\n".join([markdown_text] + field_blocks)
         for summary in _build_figure_summaries(content):
-            rendered_summary = _render_figure_summary(summary, markdown_text)
+            rendered_summary = _render_figure_summary(summary, emitted_text)
             if rendered_summary:
                 text_blocks.append(rendered_summary)
 
-    return "\n\n".join(block for block in text_blocks if block).strip()
+    description_text = "\n\n".join(block for block in text_blocks if block).strip()
+    debug_print(
+        f"[CONTENT_UNDERSTANDING] Image analyzer {config['image_analyzer_id']} returned "
+        f"{len(description_text)} character(s) for {os.path.basename(image_path)}"
+    )
+    return description_text
 
 
 def test_content_understanding_connection(config_override, sample_file_path=None):