Index images without OCR text and route vision analysis through its AI connection

application/external_apps/databaseseeder/artifacts/admin_settings.json

@@ -37,7 +37,7 @@
     "enable_image_generation": false,
     "enable_image_gen_apim": false,
     "azure_openai_image_gen_endpoint": "",
-    "azure_openai_image_gen_api_version": "2024-05-01-preview",
+    "azure_openai_image_gen_api_version": "2025-04-01-preview",
     "azure_openai_image_gen_authentication_type": "key",
     "azure_openai_image_gen_subscription_id": "",
     "azure_openai_image_gen_resource_group": "",

application/single_app/config.py

@@ -98,7 +98,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.046"
+VERSION = "0.261.047"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 # Opt-out for deployments where App Service Easy Auth is active but the platform

application/single_app/functions_content.py

@@ -621,19 +621,53 @@
     return functions_settings.resolve_enhanced_extraction_engine(settings)
 
 
+# Persisted as the extraction engine reason and shown in workspace tooltips, so keep it short.
+CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON = (
+    'Content Understanding found no text, so its image analyzer description was indexed'
+)
+
+
+def _describe_image_with_content_understanding(file_path, settings, analyze_image):
+    """Return the Content Understanding image analyzer description, or '' when it has none."""
+    try:
+        description_text = str(analyze_image(file_path, settings=settings) or '').strip()
+    except Exception as image_analyzer_error:
+        log_event(
+            f"[EXTRACTION_ENGINE] Content Understanding image analyzer failed for "
+            f"{os.path.basename(file_path)}: {image_analyzer_error}. "
+            "Falling back to Document Intelligence Layout.",
+            level=logging.WARNING,
+        )
+        return ''
+
+    if not description_text:
+        log_event(
+            f"[EXTRACTION_ENGINE] Content Understanding image analyzer returned no description for "
+            f"{os.path.basename(file_path)}; falling back to Document Intelligence Layout.",
+            level=logging.WARNING,
+        )
+    return description_text
+
+
 def extract_content_with_extraction_engine(
     file_path,
     extraction_mode,
     extraction_engine=None,
     settings=None,
     pages=None,
+    is_image=False,
 ):
     """Extract page content with the engine that backs the requested extraction mode.
 
     Standard extraction always uses Document Intelligence ``prebuilt-read``. Enhanced extraction
     prefers Azure AI Content Understanding, but always falls back to Document Intelligence
     ``prebuilt-layout`` so a Content Understanding outage never blocks ingestion.
 
+    When ``is_image`` is set and the document analyzer finds no content, which is normal for a
+    photo with no text, the Content Understanding image analyzer's description is returned as
+    the image's single page instead. Only if that also yields nothing does extraction fall back
+    to Document Intelligence Layout.
+
     Returns ``(pages_data, engine_used, fallback_reason)``.
     """
     resolved_settings = settings if settings is not None else get_settings()
@@ -650,7 +684,10 @@
             '',
         )
 
-    from functions_content_understanding import extract_content_with_content_understanding
+    from functions_content_understanding import (
+        analyze_image_with_content_understanding,
+        extract_content_with_content_understanding,
+    )
 
     try:
         content_understanding_pages = extract_content_with_content_understanding(
@@ -664,6 +701,18 @@
                 functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING,
                 '',
             )
+        if is_image:
+            image_description = _describe_image_with_content_understanding(
+                file_path,
+                resolved_settings,
+                analyze_image_with_content_understanding,
+            )
+            if image_description:
+                return (
+                    [{'page_number': 1, 'content': image_description}],
+                    functions_settings.EXTRACTION_ENGINE_CONTENT_UNDERSTANDING,
+                    CONTENT_UNDERSTANDING_IMAGE_ANALYZER_REASON,
+                )
         fallback_reason = (
             'Content Understanding returned no content, so Document Intelligence Layout was used'
         )

application/single_app/functions_content_understanding.py

@@ -14,6 +14,7 @@
 import json
 import logging
 import os
+import re
 import time
 
 import requests
@@ -48,6 +49,11 @@
     "Understanding defaults before using Enhanced extraction."
 )
 
+# Image analyzers such as prebuilt-imageSearch return a bare image reference, for example
+# "![image](pages/1)", as the markdown and put the actual description in a Summary field.
+MARKDOWN_IMAGE_PLACEHOLDER_PATTERN = re.compile(r'!\[[^\]]*\]\([^)]*\)')
+CONTENT_UNDERSTANDING_SUMMARY_FIELD = 'summary'
+
 
 class ContentUnderstandingError(Exception):
     """Raised when a Content Understanding request cannot be completed."""
@@ -521,8 +527,45 @@
     return pages_data
 
 
+def _is_image_placeholder_markdown(markdown_text):
+    """Return True when markdown holds nothing but image references such as ``![image](pages/1)``."""
+    return not MARKDOWN_IMAGE_PLACEHOLDER_PATTERN.sub('', str(markdown_text or '')).strip()
+
+
+def _build_string_field_blocks(content):
+    """Return readable text for the non-empty string fields an analyzer extracted.
+
+    The ``Summary`` field is emitted as plain text first, because it is the analyzer's own
+    description of the input. Every other string field is emitted as ``Name: value``.
+    """
+    fields = content.get('fields')
+    if not isinstance(fields, dict):
+        return []
+
+    summary_blocks = []
+    field_blocks = []
+    for field_name, field_value in fields.items():
+        if not isinstance(field_value, dict):
+            continue
+        value_string = field_value.get('valueString')
+        if not isinstance(value_string, str) or not value_string.strip():
+            continue
+
+        if str(field_name).strip().lower() == CONTENT_UNDERSTANDING_SUMMARY_FIELD:
+            summary_blocks.append(value_string.strip())
+        else:
+            field_blocks.append(f"{field_name}: {value_string.strip()}")
+
+    return summary_blocks + field_blocks
+
+
 def analyze_image_with_content_understanding(image_path, settings=None):
-    """Analyze a standalone image and return its description text, or an empty string."""
+    """Analyze a standalone image and return its description text, or an empty string.
+
+    Image analyzers describe the image in extracted fields rather than in the markdown, which is
+    only a placeholder image reference, so string fields are read alongside any real markdown
+    and figure descriptions.
+    """
     resolved_settings = settings if settings is not None else _get_settings()
     config = _resolve_config(settings=resolved_settings)
 
@@ -535,15 +578,27 @@
     text_blocks = []
     for content in _iter_document_contents(result):
         markdown_text = str(content.get('markdown') or '').strip()
+        if _is_image_placeholder_markdown(markdown_text):
+            markdown_text = ''
         if markdown_text:
             text_blocks.append(markdown_text)
 
+        field_blocks = _build_string_field_blocks(content)
+        text_blocks.extend(field_blocks)
+
+        # De-duplicate figure descriptions against everything already emitted for this content.
+        emitted_text = "\n\n".join([markdown_text] + field_blocks)
         for summary in _build_figure_summaries(content):
-            rendered_summary = _render_figure_summary(summary, markdown_text)
+            rendered_summary = _render_figure_summary(summary, emitted_text)
             if rendered_summary:
                 text_blocks.append(rendered_summary)
 
-    return "\n\n".join(block for block in text_blocks if block).strip()
+    description_text = "\n\n".join(block for block in text_blocks if block).strip()
+    debug_print(
+        f"[CONTENT_UNDERSTANDING] Image analyzer {config['image_analyzer_id']} returned "
+        f"{len(description_text)} character(s) for {os.path.basename(image_path)}"
+    )
+    return description_text
 
 
 def test_content_understanding_connection(config_override, sample_file_path=None):