diff --git a/application/single_app/config.py b/application/single_app/config.py index 36591a2e6..4b44e54ca 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,9 +97,17 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.003" +VERSION = "0.261.007" IS_DEVELOPMENT = is_development_env_enabled() +# Opt-out for deployments where App Service Easy Auth is active but the platform +# /.auth/logout endpoint is not reachable on the public host (for example, when a +# custom domain or gateway does not route /.auth/* to the App Service origin). +DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT = os.getenv( + 'DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT', + '' +).strip().lower() == 'true' + SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false' SESSION_COOKIE_SECURE = os.getenv('SESSION_COOKIE_SECURE', 'false').lower() == 'true' diff --git a/application/single_app/example.env b/application/single_app/example.env index 7803e0b38..ca4e052c2 100644 --- a/application/single_app/example.env +++ b/application/single_app/example.env @@ -20,4 +20,12 @@ AZURE_ENVIRONMENT="public" # Optional Graph overrides (for cross-cloud identity/Graph scenarios) # Example values: # CUSTOM_GRAPH_URL_VALUE="https://graph.microsoft.com" -# CUSTOM_GRAPH_AUTHORITY_URL_VALUE="https://login.microsoftonline.com" \ No newline at end of file +# CUSTOM_GRAPH_AUTHORITY_URL_VALUE="https://login.microsoftonline.com" + +# Logout behavior on Azure App Service +# SimpleChat detects App Service Easy Auth from the X-MS-CLIENT-PRINCIPAL request headers +# the platform injects, and routes logout through /.auth/logout so the platform session is +# cleared. Set this to "true" only if Easy Auth is active but /.auth/logout is not reachable +# on your public host, for example when a custom domain or gateway does not route /.auth/* +# to the App Service origin. Has no effect when running locally. +# DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT="true" diff --git a/application/single_app/functions_data_management_search_write_fence.py b/application/single_app/functions_data_management_search_write_fence.py index 3e83b24e2..96c06f48f 100644 --- a/application/single_app/functions_data_management_search_write_fence.py +++ b/application/single_app/functions_data_management_search_write_fence.py @@ -2,6 +2,7 @@ """Cross-app Azure AI Search write fencing for Data Management migrations.""" import copy +import threading import time import uuid from contextlib import contextmanager @@ -25,6 +26,7 @@ DATA_MANAGEMENT_TARGET_MIGRATION_COORDINATOR_TYPE = ( "data_management_target_migration_coordinator" ) +DATA_MANAGEMENT_SEARCH_WRITE_GATE_LOCAL_LOCK = threading.Lock() class DataManagementSearchWriteGateError(RuntimeError): @@ -186,7 +188,8 @@ def _open_expired_gate(container, gate, now=None): def acquire_data_management_search_write_slot(container): """Reserve one bounded target Search write before issuing the data-plane request.""" - for _attempt in range(12): + deadline = time.monotonic() + DATA_MANAGEMENT_SEARCH_WRITE_REQUEST_TIMEOUT_SECONDS + while time.monotonic() < deadline: now = _now_utc() gate = _create_or_read_gate(container) if gate.get("type") != DATA_MANAGEMENT_SEARCH_WRITE_GATE_TYPE: @@ -199,6 +202,7 @@ def acquire_data_management_search_write_slot(container): ) if gate.get("state") != DATA_MANAGEMENT_SEARCH_WRITE_GATE_STATE_OPEN: if _open_expired_gate(container, gate, now) is None: + time.sleep(DATA_MANAGEMENT_SEARCH_WRITE_GATE_POLL_SECONDS) continue continue lease_token = uuid.uuid4().hex @@ -217,8 +221,9 @@ def acquire_data_management_search_write_slot(container): }) if _replace_gate(container, gate, replacement) is not None: return lease_token + time.sleep(DATA_MANAGEMENT_SEARCH_WRITE_GATE_POLL_SECONDS) raise DataManagementSearchWriteGateError( - "The Data Management Search write gate changed too often to reserve a write slot." + "The Data Management Search write gate could not reserve a write slot before the request timeout." ) @@ -251,14 +256,15 @@ def release_data_management_search_write_slot(container, lease_token): @contextmanager def hold_data_management_search_write_slot(container): """Hold a target Search write slot until a response is known or its ambiguity lease expires.""" - lease_token = acquire_data_management_search_write_slot(container) - response_confirmed = False - try: - yield - response_confirmed = True - finally: - if response_confirmed: - release_data_management_search_write_slot(container, lease_token) + with DATA_MANAGEMENT_SEARCH_WRITE_GATE_LOCAL_LOCK: + lease_token = acquire_data_management_search_write_slot(container) + response_confirmed = False + try: + yield + response_confirmed = True + finally: + if response_confirmed: + release_data_management_search_write_slot(container, lease_token) def acquire_data_management_search_write_fence( diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index 346617244..a055c13b4 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -3,6 +3,7 @@ import re import shutil import subprocess +import time import traceback import zipfile from io import BytesIO @@ -45,6 +46,11 @@ class DocumentSearchAclProjectionDeferredError(RuntimeError): """Raised when an authorization-reducing Search ACL update must be retried safely.""" +MARKDOWN_ORDERED_DICT_MUTATION_MESSAGE = "OrderedDict mutated during iteration" +MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS = 2 +MARKDOWN_ORDERED_DICT_RETRY_DELAY_SECONDS = 0.5 + + def _search_indexing_results_succeeded(results): """Return whether Azure AI Search acknowledged every requested document mutation.""" normalized_results = list(results or []) @@ -6598,7 +6604,7 @@ def process_md(document_id, user_id, temp_file_path, original_filename, enable_e if current_word_count >= min_chunk_words or i == len(initial_chunks_content) - 1: # If the combined chunk meets min size OR it's the last chunk, save it if current_chunk_text.strip(): - final_chunks.append(current_chunk_text) + final_chunks.append(current_chunk_text) buffer_chunk = "" # Reset buffer else: # Accumulate in buffer if below min size and not the last chunk @@ -6612,32 +6618,36 @@ def process_md(document_id, user_id, temp_file_path, original_filename, enable_e num_chunks_final = len(final_chunks) update_callback(number_of_pages=num_chunks_final) - for idx, chunk_content in enumerate(final_chunks, start=1): - update_callback( - current_file_chunk=idx, - status=f"Saving chunk {idx}/{num_chunks_final}..." - ) - args = { + all_chunks = [] + for chunk_content in final_chunks: + if not chunk_content or not chunk_content.strip(): + continue + all_chunks.append({ "page_text_content": chunk_content, - "page_number": idx, + "page_number": len(all_chunks) + 1, "file_name": original_filename, - "user_id": user_id, - "document_id": document_id - } - - if is_public_workspace: - args["public_workspace_id"] = public_workspace_id - elif is_group: - args["group_id"] = group_id + }) - token_usage = save_chunks(**args) - total_chunks_saved += 1 + if all_chunks: + if len(all_chunks) != num_chunks_final: + num_chunks_final = len(all_chunks) + update_callback(number_of_pages=num_chunks_final) + update_callback( + current_file_chunk=1, + status=f"Batch saving {num_chunks_final} Markdown chunk(s)..." + ) + token_usage = save_chunks_batch( + all_chunks, + user_id, + document_id, + group_id=group_id, + public_workspace_id=public_workspace_id + ) + total_chunks_saved = len(all_chunks) - # Accumulate embedding tokens if token_usage: - total_embedding_tokens += token_usage.get('total_tokens', 0) - if not embedding_model_name: - embedding_model_name = token_usage.get('model_deployment_name') + total_embedding_tokens = token_usage.get('total_tokens', 0) + embedding_model_name = token_usage.get('model_deployment_name') except Exception as e: raise Exception(f"Failed processing Markdown file {original_filename}: {e}") @@ -9042,6 +9052,50 @@ def _resolve_processing_complete_status(total_chunks_saved, file_ext, image_exte return "Processing complete" + +def _is_markdown_ordered_dict_mutation_error(exc): + """Return whether Markdown processing hit the transient OrderedDict mutation failure.""" + return MARKDOWN_ORDERED_DICT_MUTATION_MESSAGE in str(exc or "") + + +def _process_markdown_with_ordered_dict_retry(processor_args, update_callback): + """Retry Markdown processing when the parser hits a transient OrderedDict mutation.""" + for attempt in range(MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS + 1): + try: + return process_md(**{k: v for k, v in processor_args.items() if k != "file_ext"}) + except Exception as exc: + if ( + not _is_markdown_ordered_dict_mutation_error(exc) or + attempt >= MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS + ): + raise + + retry_number = attempt + 1 + original_filename = processor_args.get("original_filename") + document_id = processor_args.get("document_id") + log_event( + "[DOCUMENTS] Retrying Markdown processing after transient OrderedDict mutation.", + extra={ + "document_id": document_id, + "file_name": original_filename, + "retry_number": retry_number, + "max_retries": MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS, + "error_type": type(exc).__name__, + }, + level=logging.WARNING, + exceptionTraceback=True, + ) + update_callback( + status=( + "Retrying Markdown processing after a transient parser concurrency error " + f"({retry_number}/{MARKDOWN_ORDERED_DICT_RETRY_ATTEMPTS})..." + ) + ) + time.sleep(MARKDOWN_ORDERED_DICT_RETRY_DELAY_SECONDS * retry_number) + + raise RuntimeError("Markdown processing retry loop exited unexpectedly.") + + def process_document_upload_background(document_id, user_id, temp_file_path, original_filename, group_id=None, public_workspace_id=None, extraction_mode_override=None): """ Main background task dispatcher for document processing. @@ -9167,7 +9221,10 @@ def update_doc_callback(**kwargs): else: total_chunks_saved = result elif file_ext == '.md': - result = process_md(**{k: v for k, v in processor_args_without_auto_metadata.items() if k != "file_ext"}) + result = _process_markdown_with_ordered_dict_retry( + processor_args_without_auto_metadata, + update_doc_callback, + ) if isinstance(result, tuple) and len(result) == 3: total_chunks_saved, total_embedding_tokens, embedding_model_name = result else: diff --git a/application/single_app/gunicorn.conf.py b/application/single_app/gunicorn.conf.py index eed906759..24e35571b 100644 --- a/application/single_app/gunicorn.conf.py +++ b/application/single_app/gunicorn.conf.py @@ -31,6 +31,9 @@ def _env_bool(name, default): # Request-count recycling can terminate in-process background exports mid-batch. max_requests = _env_int('GUNICORN_MAX_REQUESTS', 0 if background_tasks_enabled else 500) max_requests_jitter = _env_int('GUNICORN_MAX_REQUESTS_JITTER', 0 if max_requests == 0 else 50) +# Azure AD auth callback query strings can exceed Gunicorn's default (4094). +# Use the Gunicorn maximum by default, while still allowing override via env var. +limit_request_line = _env_int('GUNICORN_LIMIT_REQUEST_LINE', 8190) accesslog = '-' errorlog = '-' capture_output = True diff --git a/application/single_app/route_frontend_authentication.py b/application/single_app/route_frontend_authentication.py index c07b51519..db795d249 100644 --- a/application/single_app/route_frontend_authentication.py +++ b/application/single_app/route_frontend_authentication.py @@ -6,6 +6,7 @@ import requests from config import * +from config import DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT from functions_activity_logging import log_user_login, record_user_login_session_activity from functions_terms_of_use import ( apply_pending_pre_auth_terms_of_use, @@ -42,16 +43,36 @@ def build_front_door_urls(front_door_url): def _use_app_service_easy_auth_logout(): - """Return True when the current request is running behind App Service Easy Auth.""" + """ + Determine whether logout should route through the App Service Easy Auth endpoint. + + Args: + None. + + Returns: + bool: True when the current request is being served behind App Service Easy Auth. + Raises: + None. + """ + if DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT: + debug_print("Easy Auth logout disabled by DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT; using local logout.") + return False + if not os.getenv('WEBSITE_HOSTNAME'): return False + # Easy Auth injects these headers only on requests it actually intercepts, so they are + # the reliable per-request signal that /.auth/logout is being served for this host. easy_auth_headers = ( request.headers.get('X-MS-CLIENT-PRINCIPAL'), request.headers.get('X-MS-CLIENT-PRINCIPAL-ID'), request.headers.get('X-MS-CLIENT-PRINCIPAL-NAME'), ) - return any(easy_auth_headers) or bool(os.getenv('WEBSITE_AUTH_AAD_ALLOWED_TENANTS')) + if not any(easy_auth_headers): + debug_print("No App Service Easy Auth principal headers on this request; using local logout.") + return False + + return True def _build_app_service_easy_auth_logout_url(): diff --git a/application/single_app/static/js/public/public_workspace.js b/application/single_app/static/js/public/public_workspace.js index 3893d4715..3410be79a 100644 --- a/application/single_app/static/js/public/public_workspace.js +++ b/application/single_app/static/js/public/public_workspace.js @@ -1980,6 +1980,29 @@ async function onPublicUploadClick() { let completed = 0; let failed = 0; + function updatePublicUploadRequestSummary() { + if (uploadStatus) uploadStatus.textContent = `Queued ${completed}/${files.length}${failed ? `, Upload requests not confirmed: ${failed}` : ''}`; + } + + function finishPublicUploadRequests() { + fileInput.value = ''; + publicDocsCurrentPage = 1; + fetchPublicDocs(); + + if (uploadBtn) { + uploadBtn.disabled = false; + uploadBtn.textContent = 'Upload Document(s)'; + } + + if (uploadStatus) { + uploadStatus.textContent = failed + ? `Upload requests complete. Queued ${completed}/${files.length}; ${failed} request(s) did not confirm. Check the document list below for final processing status.` + : `Queued ${completed}/${files.length} file(s). Check the document list below for processing status.`; + } + + if (progressContainer) progressContainer.innerHTML = ''; + } + // Helper to create a unique ID for each file function makeId(file) { return 'progress-' + Math.random().toString(36).slice(2, 10) + '-' + encodeURIComponent(file.name.replace(/\W+/g, '')); @@ -2034,7 +2057,7 @@ async function onPublicUploadClick() { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Uploaded ${file.name} (100%)`; + statusText.textContent = `Queued ${file.name} (100%)`; } completed++; } else { @@ -2044,26 +2067,13 @@ async function onPublicUploadClick() { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; } - // Update summary status - if (uploadStatus) uploadStatus.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updatePublicUploadRequestSummary(); if (completed + failed === files.length) { - fileInput.value = ''; - publicDocsCurrentPage = 1; - fetchPublicDocs(); - - // Re-enable upload button if it exists - if (uploadBtn) { - uploadBtn.disabled = false; - uploadBtn.textContent = 'Upload Document(s)'; - } - - // Clear upload progress bars after all uploads and table refresh - const progressContainer = document.getElementById('public-upload-progress-container'); - if (progressContainer) progressContainer.innerHTML = ''; + finishPublicUploadRequests(); } }; @@ -2074,24 +2084,12 @@ async function onPublicUploadClick() { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; - if (uploadStatus) uploadStatus.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updatePublicUploadRequestSummary(); if (completed + failed === files.length) { - fileInput.value = ''; - publicDocsCurrentPage = 1; - fetchPublicDocs(); - - // Re-enable upload button if it exists - if (uploadBtn) { - uploadBtn.disabled = false; - uploadBtn.textContent = 'Upload Document(s)'; - } - - // Clear upload progress bars after all uploads and table refresh - const progressContainer = document.getElementById('public-upload-progress-container'); - if (progressContainer) progressContainer.innerHTML = ''; + finishPublicUploadRequests(); } }; diff --git a/application/single_app/static/js/workspace/workspace-documents.js b/application/single_app/static/js/workspace/workspace-documents.js index a887786f5..92c1de3f3 100644 --- a/application/single_app/static/js/workspace/workspace-documents.js +++ b/application/single_app/static/js/workspace/workspace-documents.js @@ -1481,6 +1481,20 @@ async function uploadWorkspaceFiles(files) { let completed = 0; let failed = 0; + function updateWorkspaceUploadRequestSummary() { + uploadStatusSpan.textContent = `Queued ${completed}/${files.length}${failed ? `, Upload requests not confirmed: ${failed}` : ''}`; + } + + function finishWorkspaceUploadRequests() { + fileInput.value = ''; + docsCurrentPage = 1; + fetchUserDocuments(); + uploadStatusSpan.textContent = failed + ? `Upload requests complete. Queued ${completed}/${files.length}; ${failed} request(s) did not confirm. Check the document list below for final processing status.` + : `Queued ${completed}/${files.length} file(s). Check the document list below for processing status.`; + if (progressContainer) progressContainer.innerHTML = ''; + } + // Helper to create a unique ID for each file function makeId(file) { return 'progress-' + Math.random().toString(36).slice(2, 10) + '-' + encodeURIComponent(file.name.replace(/\W+/g, '')); @@ -1535,7 +1549,7 @@ async function uploadWorkspaceFiles(files) { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Uploaded ${file.name} (100%)`; + statusText.textContent = `Queued ${file.name} (100%)`; } completed++; } else { @@ -1545,18 +1559,13 @@ async function uploadWorkspaceFiles(files) { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; } - // Update summary status - uploadStatusSpan.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updateWorkspaceUploadRequestSummary(); if (completed + failed === files.length) { - fileInput.value = ''; - docsCurrentPage = 1; - fetchUserDocuments(); - // Clear upload progress bars after all uploads and table refresh - if (progressContainer) progressContainer.innerHTML = ''; + finishWorkspaceUploadRequests(); } }; @@ -1567,15 +1576,12 @@ async function uploadWorkspaceFiles(files) { progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; - uploadStatusSpan.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updateWorkspaceUploadRequestSummary(); if (completed + failed === files.length) { - fileInput.value = ''; - docsCurrentPage = 1; - fetchUserDocuments(); - if (progressContainer) progressContainer.innerHTML = ''; + finishWorkspaceUploadRequests(); } }; diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index 12e2b08cf..68b120eaa 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -3404,6 +3404,20 @@ let completed = 0; let failed = 0; + function updateGroupUploadRequestSummary() { + groupUploadStatusSpan.textContent = `Queued ${completed}/${files.length}${failed ? `, Upload requests not confirmed: ${failed}` : ''}`; + } + + function finishGroupUploadRequests() { + groupFileInput.value = ''; + groupDocsCurrentPage = 1; + fetchGroupDocuments(); + groupUploadStatusSpan.textContent = failed + ? `Upload requests complete. Queued ${completed}/${files.length}; ${failed} request(s) did not confirm. Check the document list below for final processing status.` + : `Queued ${completed}/${files.length} file(s). Check the document list below for processing status.`; + if (progressContainer) progressContainer.innerHTML = ''; + } + // Helper to create a unique ID for each file function makeId(file) { return 'group-progress-' + Math.random().toString(36).slice(2, 10) + '-' + encodeURIComponent(file.name.replace(/\W+/g, '')); @@ -3458,7 +3472,7 @@ progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Uploaded ${file.name} (100%)`; + statusText.textContent = `Queued ${file.name} (100%)`; } completed++; } else { @@ -3468,18 +3482,13 @@ progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; } - // Update summary status - groupUploadStatusSpan.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updateGroupUploadRequestSummary(); if (completed + failed === files.length) { - groupFileInput.value = ''; - groupDocsCurrentPage = 1; - fetchGroupDocuments(); - // Clear upload progress bars after all uploads and table refresh - if (progressContainer) progressContainer.innerHTML = ''; + finishGroupUploadRequests(); } }; @@ -3490,15 +3499,12 @@ progressBar.classList.remove('progress-bar-animated'); } if (statusText) { - statusText.textContent = `Failed to upload ${file.name}`; + statusText.textContent = `Upload request did not confirm for ${file.name}`; } failed++; - groupUploadStatusSpan.textContent = `Uploaded ${completed}/${files.length}${failed ? `, Failed: ${failed}` : ''}`; + updateGroupUploadRequestSummary(); if (completed + failed === files.length) { - groupFileInput.value = ''; - groupDocsCurrentPage = 1; - fetchGroupDocuments(); - if (progressContainer) progressContainer.innerHTML = ''; + finishGroupUploadRequests(); } }; @@ -6846,6 +6852,22 @@