Fix large workspace upload reliability and status reporting

application/single_app/config.py

@@ -97,9 +97,17 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.261.003"
+VERSION = "0.261.007"
 IS_DEVELOPMENT = is_development_env_enabled()
 
+# Opt-out for deployments where App Service Easy Auth is active but the platform
+# /.auth/logout endpoint is not reachable on the public host (for example, when a
+# custom domain or gateway does not route /.auth/* to the App Service origin).
+DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT = os.getenv(
+    'DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT',
+    ''
+).strip().lower() == 'true'
+
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
 SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false'
 SESSION_COOKIE_SECURE = os.getenv('SESSION_COOKIE_SECURE', 'false').lower() == 'true'

application/single_app/example.env

@@ -20,4 +20,12 @@
 # Optional Graph overrides (for cross-cloud identity/Graph scenarios)
 # Example values:
 # CUSTOM_GRAPH_URL_VALUE="https://graph.microsoft.com"
-# CUSTOM_GRAPH_AUTHORITY_URL_VALUE="https://login.microsoftonline.com"
+# CUSTOM_GRAPH_AUTHORITY_URL_VALUE="https://login.microsoftonline.com"
+
+# Logout behavior on Azure App Service
+# SimpleChat detects App Service Easy Auth from the X-MS-CLIENT-PRINCIPAL request headers
+# the platform injects, and routes logout through /.auth/logout so the platform session is
+# cleared. Set this to "true" only if Easy Auth is active but /.auth/logout is not reachable
+# on your public host, for example when a custom domain or gateway does not route /.auth/*
+# to the App Service origin. Has no effect when running locally.
+# DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT="true"

application/single_app/functions_data_management_search_write_fence.py

@@ -2,6 +2,7 @@
 """Cross-app Azure AI Search write fencing for Data Management migrations."""
 
 import copy
+import threading
 import time
 import uuid
 from contextlib import contextmanager
@@ -25,6 +26,7 @@
 DATA_MANAGEMENT_TARGET_MIGRATION_COORDINATOR_TYPE = (
     "data_management_target_migration_coordinator"
 )
+DATA_MANAGEMENT_SEARCH_WRITE_GATE_LOCAL_LOCK = threading.Lock()
 
 
 class DataManagementSearchWriteGateError(RuntimeError):
@@ -186,7 +188,8 @@
 
 def acquire_data_management_search_write_slot(container):
     """Reserve one bounded target Search write before issuing the data-plane request."""
-    for _attempt in range(12):
+    deadline = time.monotonic() + DATA_MANAGEMENT_SEARCH_WRITE_REQUEST_TIMEOUT_SECONDS
+    while time.monotonic() < deadline:
         now = _now_utc()
         gate = _create_or_read_gate(container)
         if gate.get("type") != DATA_MANAGEMENT_SEARCH_WRITE_GATE_TYPE:
@@ -199,6 +202,7 @@
             )
         if gate.get("state") != DATA_MANAGEMENT_SEARCH_WRITE_GATE_STATE_OPEN:
             if _open_expired_gate(container, gate, now) is None:
+                time.sleep(DATA_MANAGEMENT_SEARCH_WRITE_GATE_POLL_SECONDS)
                 continue
             continue
         lease_token = uuid.uuid4().hex
@@ -217,8 +221,9 @@
         })
         if _replace_gate(container, gate, replacement) is not None:
             return lease_token
+        time.sleep(DATA_MANAGEMENT_SEARCH_WRITE_GATE_POLL_SECONDS)
     raise DataManagementSearchWriteGateError(
-        "The Data Management Search write gate changed too often to reserve a write slot."
+        "The Data Management Search write gate could not reserve a write slot before the request timeout."
     )
 
 
@@ -251,14 +256,15 @@
 @contextmanager
 def hold_data_management_search_write_slot(container):
     """Hold a target Search write slot until a response is known or its ambiguity lease expires."""
-    lease_token = acquire_data_management_search_write_slot(container)
-    response_confirmed = False
-    try:
-        yield
-        response_confirmed = True
-    finally:
-        if response_confirmed:
-            release_data_management_search_write_slot(container, lease_token)
+    with DATA_MANAGEMENT_SEARCH_WRITE_GATE_LOCAL_LOCK:
+        lease_token = acquire_data_management_search_write_slot(container)
+        response_confirmed = False
+        try:
+            yield
+            response_confirmed = True
+        finally:
+            if response_confirmed:
+                release_data_management_search_write_slot(container, lease_token)
 
 
 def acquire_data_management_search_write_fence(

application/single_app/functions_documents.py

@@ -3,6 +3,7 @@
 import re
 import shutil
 import subprocess
+import time
 import traceback
 import zipfile
 from io import BytesIO
@@ -45,6 +46,11 @@ class DocumentSearchAclProjectionDeferredError(RuntimeError):
     """Raised when an authorization-reducing Search ACL update must be retried safely."""
 
 
+MARKDOWN_ORDERED_DICT_MUTATION_MESSAGE = "OrderedDict mutated during iteration"
+MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS = 2
+MARKDOWN_ORDERED_DICT_RETRY_DELAY_SECONDS = 0.5
+
+
 def _search_indexing_results_succeeded(results):
     """Return whether Azure AI Search acknowledged every requested document mutation."""
     normalized_results = list(results or [])
@@ -6598,7 +6604,7 @@ def process_md(document_id, user_id, temp_file_path, original_filename, enable_e
             if current_word_count >= min_chunk_words or i == len(initial_chunks_content) - 1:
                  # If the combined chunk meets min size OR it's the last chunk, save it
                 if current_chunk_text.strip():
-                     final_chunks.append(current_chunk_text)
+                    final_chunks.append(current_chunk_text)
                 buffer_chunk = "" # Reset buffer
             else:
                 # Accumulate in buffer if below min size and not the last chunk
@@ -6612,32 +6618,36 @@ def process_md(document_id, user_id, temp_file_path, original_filename, enable_e
         num_chunks_final = len(final_chunks)
         update_callback(number_of_pages=num_chunks_final)
 
-        for idx, chunk_content in enumerate(final_chunks, start=1):
-            update_callback(
-                current_file_chunk=idx,
-                status=f"Saving chunk {idx}/{num_chunks_final}..."
-            )
-            args = {
+        all_chunks = []
+        for chunk_content in final_chunks:
+            if not chunk_content or not chunk_content.strip():
+                continue
+            all_chunks.append({
                 "page_text_content": chunk_content,
-                "page_number": idx,
+                "page_number": len(all_chunks) + 1,
                 "file_name": original_filename,
-                "user_id": user_id,
-                "document_id": document_id
-            }
-
-            if is_public_workspace:
-                args["public_workspace_id"] = public_workspace_id
-            elif is_group:
-                args["group_id"] = group_id
+            })
 
-            token_usage = save_chunks(**args)
-            total_chunks_saved += 1
+        if all_chunks:
+            if len(all_chunks) != num_chunks_final:
+                num_chunks_final = len(all_chunks)
+                update_callback(number_of_pages=num_chunks_final)
+            update_callback(
+                current_file_chunk=1,
+                status=f"Batch saving {num_chunks_final} Markdown chunk(s)..."
+            )
+            token_usage = save_chunks_batch(
+                all_chunks,
+                user_id,
+                document_id,
+                group_id=group_id,
+                public_workspace_id=public_workspace_id
+            )
+            total_chunks_saved = len(all_chunks)
 
-            # Accumulate embedding tokens
             if token_usage:
-                total_embedding_tokens += token_usage.get('total_tokens', 0)
-                if not embedding_model_name:
-                    embedding_model_name = token_usage.get('model_deployment_name')
+                total_embedding_tokens = token_usage.get('total_tokens', 0)
+                embedding_model_name = token_usage.get('model_deployment_name')
 
     except Exception as e:
         raise Exception(f"Failed processing Markdown file {original_filename}: {e}")
@@ -9042,6 +9052,50 @@ def _resolve_processing_complete_status(total_chunks_saved, file_ext, image_exte
 
     return "Processing complete"
 
+
+def _is_markdown_ordered_dict_mutation_error(exc):
+    """Return whether Markdown processing hit the transient OrderedDict mutation failure."""
+    return MARKDOWN_ORDERED_DICT_MUTATION_MESSAGE in str(exc or "")
+
+
+def _process_markdown_with_ordered_dict_retry(processor_args, update_callback):
+    """Retry Markdown processing when the parser hits a transient OrderedDict mutation."""
+    for attempt in range(MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS + 1):
+        try:
+            return process_md(**{k: v for k, v in processor_args.items() if k != "file_ext"})
+        except Exception as exc:
+            if (
+                not _is_markdown_ordered_dict_mutation_error(exc) or
+                attempt >= MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS
+            ):
+                raise
+
+            retry_number = attempt + 1
+            original_filename = processor_args.get("original_filename")
+            document_id = processor_args.get("document_id")
+            log_event(
+                "[DOCUMENTS] Retrying Markdown processing after transient OrderedDict mutation.",
+                extra={
+                    "document_id": document_id,
+                    "file_name": original_filename,
+                    "retry_number": retry_number,
+                    "max_retries": MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS,
+                    "error_type": type(exc).__name__,
+                },
+                level=logging.WARNING,
+                exceptionTraceback=True,
+            )
+            update_callback(
+                status=(
+                    "Retrying Markdown processing after a transient parser concurrency error "
+                    f"({retry_number}/{MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS})..."
+                )
+            )
+            time.sleep(MARKDOWN_ORDERED_DICT_RETRY_DELAY_SECONDS * retry_number)
+
+    raise RuntimeError("Markdown processing retry loop exited unexpectedly.")
+
+
 def process_document_upload_background(document_id, user_id, temp_file_path, original_filename, group_id=None, public_workspace_id=None, extraction_mode_override=None):
     """
     Main background task dispatcher for document processing.
@@ -9167,7 +9221,10 @@ def update_doc_callback(**kwargs):
             else:
                 total_chunks_saved = result
         elif file_ext == '.md':
-            result = process_md(**{k: v for k, v in processor_args_without_auto_metadata.items() if k != "file_ext"})
+            result = _process_markdown_with_ordered_dict_retry(
+                processor_args_without_auto_metadata,
+                update_doc_callback,
+            )
             if isinstance(result, tuple) and len(result) == 3:
                 total_chunks_saved, total_embedding_tokens, embedding_model_name = result
             else:

application/single_app/gunicorn.conf.py

@@ -31,6 +31,9 @@ def _env_bool(name, default):
 # Request-count recycling can terminate in-process background exports mid-batch.
 max_requests = _env_int('GUNICORN_MAX_REQUESTS', 0 if background_tasks_enabled else 500)
 max_requests_jitter = _env_int('GUNICORN_MAX_REQUESTS_JITTER', 0 if max_requests == 0 else 50)
+# Azure AD auth callback query strings can exceed Gunicorn's default (4094).
+# Use the Gunicorn maximum by default, while still allowing override via env var.
+limit_request_line = _env_int('GUNICORN_LIMIT_REQUEST_LINE', 8190)
 accesslog = '-'
 errorlog = '-'
 capture_output = True

application/single_app/route_frontend_authentication.py

@@ -6,6 +6,7 @@
 import requests
 
 from config import *
+from config import DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT
 from functions_activity_logging import log_user_login, record_user_login_session_activity
 from functions_terms_of_use import (
     apply_pending_pre_auth_terms_of_use,
@@ -42,16 +43,36 @@ def build_front_door_urls(front_door_url):
 
 
 def _use_app_service_easy_auth_logout():
-    """Return True when the current request is running behind App Service Easy Auth."""
+    """
+    Determine whether logout should route through the App Service Easy Auth endpoint.
+
+    Args:
+        None.
+
+    Returns:
+        bool: True when the current request is being served behind App Service Easy Auth.
+    Raises:
+        None.
+    """
+    if DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT:
+        debug_print("Easy Auth logout disabled by DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT; using local logout.")
+        return False
+
     if not os.getenv('WEBSITE_HOSTNAME'):
         return False
 
+    # Easy Auth injects these headers only on requests it actually intercepts, so they are
+    # the reliable per-request signal that /.auth/logout is being served for this host.
     easy_auth_headers = (
         request.headers.get('X-MS-CLIENT-PRINCIPAL'),
         request.headers.get('X-MS-CLIENT-PRINCIPAL-ID'),
         request.headers.get('X-MS-CLIENT-PRINCIPAL-NAME'),
     )
-    return any(easy_auth_headers) or bool(os.getenv('WEBSITE_AUTH_AAD_ALLOWED_TENANTS'))
+    if not any(easy_auth_headers):
+        debug_print("No App Service Easy Auth principal headers on this request; using local logout.")
+        return False
+
+    return True
 
 
 def _build_app_service_easy_auth_logout_url():