From 7c41a13f1491dfb84b03f4365ad81276f5a85483 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Tue, 6 Oct 2026 15:25:23 -0400 Subject: [PATCH] Tie mixed-source behavior to Enhanced Citations and describe its admin settings The V2 admin page filed five mixed-source switches under Knowledge > Deep Research with only their key names as labels. Two did nothing, and the behaviors that mattered had no control at all. Mixed-source Chat and Search, follow-up continuity, and cross-format Compare now follow Enhanced Citations on every settings load and save, with a read-time SIMPLECHAT_DISABLE_MIXED_SOURCE kill switch. Relevance candidates become a described switch under Enhanced Citations (default on, one-time upgrade), and the development metrics switch moves to Application Insights. The obsolete Analyze and shadow manifest switches are retired and their code removed, Analyze All stays off and hidden, and the cross-format Compare limitation now names Enhanced Citations. Version 0.261.260. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 85 +++- application/single_app/config.py | 2 +- application/single_app/functions_settings.py | 136 +++++- .../single_app/functions_workflow_runner.py | 31 +- application/single_app/route_backend_chats.py | 54 --- .../route_frontend_admin_settings.py | 2 + .../templates/admin/_panes/citation.html | 20 + .../templates/admin/_panes/logging.html | 12 + docs/_data/app_surface.yml | 6 +- docs/_data/features.yml | 22 +- docs/admin/chat.md | 38 +- docs/admin/operations.md | 14 +- docs/admin_configuration.md | 4 +- .../features/CROSS_FORMAT_COMPARE.md | 14 +- .../features/MIXED_SOURCE_ANALYZE.md | 6 +- ...IXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md | 6 +- .../MIXED_SOURCE_CONVERSATION_CONTINUITY.md | 6 +- ...SOURCE_HARDENING_EXTRACTION_AND_ROLLOUT.md | 4 + ..._SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md | 6 +- .../features/V2_ADMIN_CHAT_SETTINGS.md | 10 +- .../fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md | 145 ++++++ docs/explanation/fixes/index.md | 1 + docs/reference/application-workflows.md | 4 +- .../test_analyze_native_saved_integration.py | 2 +- .../test_app_settings_store_consistency.py | 3 +- .../test_content_screening_settings_api.py | 1 + .../test_cross_format_compare_workflow.py | 29 +- .../test_docs_app_surface_coverage.py | 5 +- .../test_image_chat_route_integration.py | 3 +- ...st_mixed_source_chat_search_consistency.py | 34 +- ...st_mixed_source_conversation_continuity.py | 22 +- .../test_mixed_source_settings_derivation.py | 446 ++++++++++++++++++ .../test_orchestration_single_contract.py | 5 +- .../test_tabular_document_actions_workflow.py | 79 +--- .../test_v2_admin_capability_placement.py | 59 ++- functional_tests/test_workflow_loop_limits.py | 1 + ..._workflow_orchestration_limits_settings.py | 1 + .../test_v2_admin_mixed_source_settings.py | 208 ++++++++ 38 files changed, 1307 insertions(+), 219 deletions(-) create mode 100644 docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md create mode 100644 functional_tests/test_mixed_source_settings_derivation.py create mode 100644 ui_tests/test_v2_admin_mixed_source_settings.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 374c9632f..b56d481d3 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -819,6 +819,41 @@ # key is what takes it out of that scan, so these five are declared rather # than guessed at. Wording is taken from the V1 panes so both interfaces say # the same thing. + # + # Application Insights is declared for the same reason. The scan splits + # `enable_appinsights_global_logging` into "appinsights", which matches no + # section, and "logging", which matches Debug Logging, so the switch that sends + # everything to Application Insights sat under Debug Logging. The mixed-source + # telemetry switch, which the scan had filed under Deep Research on the word + # "source", reports to the same place and is declared beside it. + "application-insights-section": [ + { + "key": "enable_appinsights_global_logging", + "type": "switch", + "label": "Enable Application Insights Global Logging", + "help": ( + "Sends global logging for all agents and orchestration events to " + "Application Insights. Changing this requires an application restart " + "to take effect." + ), + "default": False, + }, + { + "key": "enable_mixed_source_development_telemetry", + "type": "switch", + "label": "Record mixed document and spreadsheet metrics", + "help": ( + "Logs aggregate counts, timings and token totals for the processing " + "behind Chat, Search, Analyze and Compare over workspace documents and " + "spreadsheets: how many sources completed, were partial, failed, were " + "skipped or were canceled, plus authorization failures and background " + "exports. Never records prompts, content, file names, document IDs or " + "storage paths. Turn it on while checking how these requests behave; " + "each request adds several log entries." + ), + "default": False, + }, + ], "health-check-section": [ { "key": "enable_external_healthcheck", @@ -2772,7 +2807,10 @@ "help": ( "Stores original files in an Azure Storage account so citations can " "link to and preview the source document rather than only quoting " - "extracted text." + "extracted text. It also turns on spreadsheet analysis: CSV and Excel " + "files are calculated over row by row instead of searched as text, and " + "Chat, Search, Analyze and Compare can combine documents and " + "spreadsheets selected together." ), "default": False, }, @@ -2824,6 +2862,22 @@ }, ], }, + { + "key": "enable_mixed_source_relevance_candidates", + "type": "switch", + "label": "Look for relevant spreadsheets when no files are selected", + "help": ( + "When a chat searches workspace documents without specific files " + "selected, run a second search aimed at spreadsheet columns and sheets " + "and keep up to six spreadsheets that fit the question. A question about " + "figures can then be answered from the right file even when its text did " + "not rank among the first results. The extra search adds a little time to " + "those messages, and a matched spreadsheet that gets analyzed adds model " + "calls." + ), + "default": True, + "depends_on": {"key": "enable_enhanced_citations", "equals": True}, + }, { "key": "tabular_preview_max_blob_size_mb", "type": "number", @@ -6321,12 +6375,33 @@ "administrator-editable." ), "enable_mixed_source_chat_search": ( - "Staged rollout flag for mixed-source chat and search, with no control in " - "the server-rendered admin form." + "Derived, not stored: get_settings() and update_settings() set it to " + "enable_enhanced_citations on every read and save, because mixed document and " + "spreadsheet Chat and Search needs the spreadsheet engine Enhanced Citations " + "provides. A switch here would revert on the next page load. The only override " + "is the SIMPLECHAT_DISABLE_MIXED_SOURCE environment kill switch." ), "enable_mixed_source_conversation_continuity": ( - "Staged rollout flag gated behind enable_mixed_source_chat_search, with no " - "control in the server-rendered admin form." + "Derived from enable_enhanced_citations in the same way as " + "enable_mixed_source_chat_search, which it extends with reauthorized " + "follow-up grounding." + ), + "enable_cross_format_compare": ( + "Derived from enable_enhanced_citations in the same way as " + "enable_mixed_source_chat_search. Comparing a document with a spreadsheet " + "needs the spreadsheet engine, and without it the request is refused rather " + "than comparing the spreadsheet as text." + ), + "enable_cross_format_compare_one_to_many": ( + "Derived from enable_enhanced_citations alongside enable_cross_format_compare. " + "The Comparison document limits under Document Actions already bound how many " + "Targets one request may include." + ), + "enable_mixed_source_analyze_all": ( + "Gates an Analyze target of every document in scope, which no chat or workflow " + "screen can request; only a hand-built API request can reach it. A switch here " + "would change nothing an administrator can see. The 0.261.260 settings upgrade " + "resets it to off once." ), } diff --git a/application/single_app/config.py b/application/single_app/config.py index 670ab4808..3a890675e 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.259" +VERSION = "0.261.260" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index c13a81c53..6e748af14 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -265,11 +265,21 @@ def normalize_public_workspace_display_settings(settings): # Switches that no longer exist. A stored value is removed on load so it is not echoed to -# the browser. Earlier releases already left it out of a saved run's settings fingerprint, -# so removing it does not change that fingerprint. +# the browser, and so the V2 admin page stops drawing a switch that changes nothing. RETIRED_SETTING_KEYS = ( - # Gather / Reason / Render became the only chat orchestration plan contract. + # Gather / Reason / Render became the only chat orchestration plan contract. Earlier + # releases already left it out of a saved run's settings fingerprint, so removing it + # does not change that fingerprint. "enable_chat_orchestration_harness", + # Combined Analyze has always routed a mixed narrative and tabular selection natively + # since 0.250.071; nothing has read this switch since then. Unlike the harness toggle it + # was part of a saved run's settings fingerprint, so removing it changes that fingerprint + # once, exactly as any administrator save does. + "enable_mixed_source_analyze", + # Phase 1 shadow manifests: resolved in the background and discarded. Analyze, Compare and + # mixed-source Chat and Search all resolve their manifests for real now. Same one-time + # fingerprint change as above. + "enable_mixed_source_manifest", ) @@ -561,23 +571,24 @@ def is_fact_memory_enabled(settings): return bool((settings or {}).get('enable_fact_memory_plugin', False)) -def is_mixed_source_manifest_enabled(settings): - """Return whether Phase 1 mixed-source manifest diagnostics are enabled.""" - return bool((settings or {}).get('enable_mixed_source_manifest', False)) - - def is_mixed_source_development_telemetry_enabled(settings): """Return whether aggregate-only mixed-source development telemetry is enabled.""" return bool((settings or {}).get('enable_mixed_source_development_telemetry', False)) def is_mixed_source_chat_search_enabled(settings): - """Return whether Phase 2 mixed-source Chat and Search behavior is enabled.""" + """Return whether mixed-source Chat and Search behavior is active. + + Derived from Enhanced Citations by normalize_mixed_source_derived_settings(). + """ return bool((settings or {}).get('enable_mixed_source_chat_search', False)) def is_mixed_source_conversation_continuity_enabled(settings): - """Return whether Phase 5 reauthorized source-continuity metadata is enabled.""" + """Return whether reauthorized source-continuity metadata is active. + + Derived from Enhanced Citations, and effective only with mixed-source Chat and Search. + """ return bool( (settings or {}).get('enable_mixed_source_chat_search', False) and (settings or {}).get('enable_mixed_source_conversation_continuity', False) @@ -585,12 +596,18 @@ def is_mixed_source_conversation_continuity_enabled(settings): def is_cross_format_compare_enabled(settings): - """Return whether Phase 4 native mixed-source Compare behavior is enabled.""" + """Return whether native mixed narrative and tabular Compare is active. + + Derived from Enhanced Citations by normalize_mixed_source_derived_settings(). + """ return bool((settings or {}).get('enable_cross_format_compare', False)) def is_cross_format_compare_one_to_many_enabled(settings): - """Return whether the separately staged one-to-many mixed-target rollout is enabled.""" + """Return whether a mixed-format Compare may have more than one Target. + + Derived from Enhanced Citations, and effective only with cross-format Compare. + """ return bool( (settings or {}).get('enable_cross_format_compare', False) and (settings or {}).get('enable_cross_format_compare_one_to_many', False) @@ -598,13 +615,93 @@ def is_cross_format_compare_one_to_many_enabled(settings): def is_mixed_source_relevance_candidates_enabled(settings): - """Return whether Phase 2 relevance-mode table candidates are enabled.""" + """Return whether relevance-mode spreadsheet candidates are enabled. + + An administrator choice, effective only while mixed-source Chat and Search is active. + """ return bool( (settings or {}).get('enable_mixed_source_chat_search', False) and (settings or {}).get('enable_mixed_source_relevance_candidates', False) ) +# Mixed-source behaviors hand CSV and Excel files to the native tabular engine, and that engine +# only exists while Enhanced Citations is on (is_tabular_processing_enabled). With it off, those +# files are indexed row by row and answered from search chunks, while the mixed-source path would +# report them as unavailable, because a table never falls back to narrative processing. So these +# are not stored choices: they follow Enhanced Citations and are rewritten on every load and save. +MIXED_SOURCE_DERIVED_SETTING_KEYS = ( + 'enable_mixed_source_chat_search', + 'enable_mixed_source_conversation_continuity', + 'enable_cross_format_compare', + 'enable_cross_format_compare_one_to_many', +) +# The only rollback path for the derived behaviors. Applied to every settings read, never stored. +MIXED_SOURCE_KILL_SWITCH_ENV = 'SIMPLECHAT_DISABLE_MIXED_SOURCE' +MIXED_SOURCE_SETTINGS_VERSION_KEY = 'mixed_source_settings_version' +MIXED_SOURCE_SETTINGS_VERSION = 1 + + +def normalize_mixed_source_derived_settings(settings): + """Set each derived mixed-source behavior to whether native spreadsheet processing exists. + + Returns True when the settings were changed. + """ + if not isinstance(settings, dict): + return False + + available = is_tabular_processing_enabled(settings) + changed = False + for key in MIXED_SOURCE_DERIVED_SETTING_KEYS: + if settings.get(key) is not available: + settings[key] = available + changed = True + return changed + + +def normalize_mixed_source_settings_upgrade(settings): + """Apply the one-time mixed-source settings upgrade. + + Before version 1 the V2 admin page drew these two switches in Deep Research with no + description, because nothing declared them: + + - ``enable_mixed_source_relevance_candidates`` had no effect at all, because the Chat and + Search behavior it extends had no control anywhere. It now defaults on, and a stored off + was never a working choice, so it is turned on. + - ``enable_mixed_source_analyze_all`` gates an Analyze target that no chat or workflow + screen can request. It is reset to off and is no longer drawn. + + The version is stored, so an administrator who turns relevance candidates off afterwards + keeps that choice. Returns True when the settings were changed. + """ + if not isinstance(settings, dict): + return False + + stored_version = settings.get(MIXED_SOURCE_SETTINGS_VERSION_KEY) + if type(stored_version) is int and stored_version >= MIXED_SOURCE_SETTINGS_VERSION: + return False + + settings['enable_mixed_source_relevance_candidates'] = True + settings['enable_mixed_source_analyze_all'] = False + settings[MIXED_SOURCE_SETTINGS_VERSION_KEY] = MIXED_SOURCE_SETTINGS_VERSION + return True + + +def _apply_mixed_source_env_kill_switch(settings_payload): + """Force the derived mixed-source behaviors off when the emergency env kill switch is set. + + They follow Enhanced Citations and have no admin switch, so this environment variable is + the only way to roll them back during an operator incident. It applies to each settings + read and is never persisted, so removing it restores the derived values immediately. + """ + if not isinstance(settings_payload, dict): + return settings_payload + if _env_flag_enabled(MIXED_SOURCE_KILL_SWITCH_ENV): + for key in MIXED_SOURCE_DERIVED_SETTING_KEYS: + settings_payload[key] = False + return settings_payload + + CHAT_FILE_UPLOAD_APP_ROLE = "ChatFileUploadUser" WORKFLOW_USER_APP_ROLE = "WorkflowUser" DOCUMENT_INTELLIGENCE_PDF_IMAGE_EXTRACTION_MODES = {"read", "layout", "auto"} @@ -1476,12 +1573,17 @@ def get_settings(use_cosmos=False, include_source=False): 'tabular_generation_systemic_failure_threshold': 0.5, 'enable_multi_agent_orchestration': False, 'enable_mixed_source_development_telemetry': False, - 'enable_mixed_source_manifest': False, + # The next four follow enable_enhanced_citations on every load and save; see + # normalize_mixed_source_derived_settings(). False matches its default. 'enable_mixed_source_chat_search': False, - 'enable_mixed_source_relevance_candidates': False, 'enable_mixed_source_conversation_continuity': False, 'enable_cross_format_compare': False, 'enable_cross_format_compare_one_to_many': False, + 'enable_mixed_source_relevance_candidates': True, + # Backend-only Analyze target with no chat or workflow control; not drawn in admin. + 'enable_mixed_source_analyze_all': False, + # One-time upgrade marker for normalize_mixed_source_settings_upgrade(). + 'mixed_source_settings_version': 0, # Chat orchestration. Deliberately prefixed `chat_orchestration_*` rather than # `orchestration_*`, because `orchestration_type` and @@ -2112,6 +2214,7 @@ def get_settings(use_cosmos=False, include_source=False): def _format_result(settings_payload, source): if isinstance(settings_payload, dict): settings_payload = _apply_tabular_parity_env_kill_switch(settings_payload) + settings_payload = _apply_mixed_source_env_kill_switch(settings_payload) if include_source: return settings_payload, source return settings_payload @@ -2165,6 +2268,8 @@ def normalize_loaded_settings(settings_item): normalize_image_connection_api_version_settings(merged) merged['enable_tabular_processing_plugin'] = is_tabular_processing_enabled(merged) + normalize_mixed_source_derived_settings(merged) + normalize_mixed_source_settings_upgrade(merged) return merged @@ -2379,6 +2484,7 @@ def apply_updates(settings_item): settings_item.get('enable_multi_model_endpoints', False), ) settings_item['enable_tabular_processing_plugin'] = is_tabular_processing_enabled(settings_item) + normalize_mixed_source_derived_settings(settings_item) return settings_item try: diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 204239f28..1d8f355b7 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -300,7 +300,6 @@ get_settings, get_user_settings, is_mixed_source_chat_search_enabled, - is_mixed_source_manifest_enabled, is_cross_format_compare_enabled, is_cross_format_compare_one_to_many_enabled, is_tabular_processing_enabled, @@ -3630,7 +3629,9 @@ def _raise_legacy_cross_format_compare_limitation(comparison_config, user_id, co partitions = partition_source_manifest(manifest) if partitions['narrative_sources'] and partitions['tabular_sources']: raise ValueError( - 'Mixed narrative and tabular Compare is temporarily unavailable while cross-format Compare is disabled.' + 'Comparing a document with a spreadsheet requires spreadsheet processing, which is ' + 'available only while Enhanced Citations is enabled. Ask an administrator to enable ' + 'Enhanced Citations, or compare files of the same kind.' ) @@ -4159,32 +4160,6 @@ def _maybe_execute_tabular_document_action( if not user_id: return None - if is_mixed_source_manifest_enabled(settings): - requested_source_ids, _ = _get_document_action_source_ids(action_config) - if requested_source_ids: - try: - resolve_authorized_source_manifest( - requested_source_ids, - user_id=user_id, - selection_mode='selected', - conversation_id=conversation_id, - active_group_ids=action_config.get('active_group_ids'), - active_public_workspace_ids=action_config.get('active_public_workspace_id'), - cancel_requested=cancel_requested, - request_correlation_id=request_correlation_id, - ) - except MixedSourceCancellationError: - raise - except Exception: - log_event( - '[MIXED_SOURCE_MANIFEST] Workflow shadow resolution failed.', - extra={ - 'requested_source_count': len(requested_source_ids), - 'selection_mode': 'selected', - }, - level=logging.WARNING, - ) - if not callable(invoke_prompt) or not is_tabular_processing_enabled(settings): return None diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 5e1207505..1804b25ed 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -866,44 +866,6 @@ def _normalize_capability_action(document_action_type): return ASSIGNED_KNOWLEDGE_USER_ACTION_SEARCH -def _maybe_resolve_chat_source_manifest( - settings, - user_id, - conversation_id, - selected_document_ids, - scope_context, -): - if not is_mixed_source_manifest_enabled(settings): - return [] - - requested_source_ids = _normalize_conversation_task_document_ids( - selected_document_ids - ) - if not requested_source_ids: - return [] - - scope_context = scope_context if isinstance(scope_context, dict) else {} - try: - return resolve_authorized_source_manifest( - requested_source_ids, - user_id=user_id, - selection_mode='selected', - conversation_id=conversation_id, - active_group_ids=scope_context.get('active_group_ids'), - active_public_workspace_ids=scope_context.get('active_public_workspace_ids'), - ) - except Exception: - log_event( - '[MIXED_SOURCE_MANIFEST] Chat shadow resolution failed.', - extra={ - 'requested_source_count': len(requested_source_ids), - 'selection_mode': 'selected', - }, - level=logging.WARNING, - ) - return [] - - def _normalize_chat_document_context_contract( settings, data, @@ -18410,14 +18372,6 @@ def result_requires_message_reload(result: Any) -> bool: xml_schema_guidance=xsd_generation_contract['guidance'], ), }) - else: - _maybe_resolve_chat_source_manifest( - settings, - user_id, - conversation_id, - effective_selected_document_ids, - scope_context, - ) request_document_context_enabled = bool( hybrid_search_enabled or ( @@ -23020,14 +22974,6 @@ def collect_stream_response_conversation_metadata(): xml_schema_guidance=xsd_generation_contract['guidance'], ), }) - else: - _maybe_resolve_chat_source_manifest( - settings, - user_id, - conversation_id, - effective_selected_document_ids, - scope_context, - ) request_document_context_enabled = bool( hybrid_search_enabled or ( diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index b13dd7517..9dd0ff73d 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -2485,6 +2485,7 @@ def is_valid_url(url): **chat_content_form_updates(form_data, settings), # Logging 'enable_appinsights_global_logging': enable_appinsights_global_logging, + 'enable_mixed_source_development_telemetry': form_data.get('enable_mixed_source_development_telemetry') == 'on', 'enable_debug_logging': enable_debug_logging, 'debug_logging_timer_enabled': debug_logging_timer_enabled, 'debug_timer_value': debug_timer_value, @@ -2770,6 +2771,7 @@ def is_valid_url(url): 'enable_enhanced_citations_mount': form_data.get('enable_enhanced_citations_mount') == 'on' and enable_enhanced_citations, 'enhanced_citations_mount': form_data.get('enhanced_citations_mount', '/view_documents').strip(), 'tabular_preview_max_blob_size_mb': int(form_data.get('tabular_preview_max_blob_size_mb', 200)), + 'enable_mixed_source_relevance_candidates': form_data.get('enable_mixed_source_relevance_candidates') == 'on', 'enable_tabular_durable_run_confirmation': form_data.get('enable_tabular_durable_run_confirmation') == 'on', 'tabular_durable_run_confirmation_threshold_rows': max(1, parse_admin_int( form_data.get('tabular_durable_run_confirmation_threshold_rows'), diff --git a/application/single_app/templates/admin/_panes/citation.html b/application/single_app/templates/admin/_panes/citation.html index 4d42286e9..ac54c337d 100644 --- a/application/single_app/templates/admin/_panes/citation.html +++ b/application/single_app/templates/admin/_panes/citation.html @@ -115,6 +115,26 @@
All filetypes
+ +
+
Spreadsheets in Chat
+

+ With Enhanced Citations on, CSV and Excel files are calculated over row by row instead of searched as text, + and Chat, Search, Analyze and Compare can combine documents and spreadsheets selected together. + Those behaviors follow this setting automatically. +

+
+ + +
+ When a chat searches workspace documents without specific files selected, run a second search aimed at spreadsheet columns and sheets + and keep up to six spreadsheets that fit the question. A question about figures can then be answered from the right file even when + its text did not rank among the first results. The extra search adds a little time to those messages, and a matched spreadsheet that + gets analyzed adds model calls. +
+
+
+
Tabular Preview Limits
diff --git a/application/single_app/templates/admin/_panes/logging.html b/application/single_app/templates/admin/_panes/logging.html index a32454471..f4361b0ca 100644 --- a/application/single_app/templates/admin/_panes/logging.html +++ b/application/single_app/templates/admin/_panes/logging.html @@ -18,6 +18,18 @@
+
+ + +
+ Logs aggregate counts, timings and token totals for the processing behind Chat, Search, Analyze and Compare over workspace + documents and spreadsheets: how many sources completed, were partial, failed, were skipped or were canceled, plus authorization + failures and background exports. Never records prompts, content, file names, document IDs or storage paths. Turn it on while + checking how these requests behave; each request adds several log entries. +
+
diff --git a/docs/_data/app_surface.yml b/docs/_data/app_surface.yml index 5e81978c6..53b4f145b 100644 --- a/docs/_data/app_surface.yml +++ b/docs/_data/app_surface.yml @@ -157,16 +157,16 @@ capabilities: default: true - key: enable_mcp_destination_governance default: false +- key: enable_mixed_source_analyze_all + default: false - key: enable_mixed_source_chat_search default: false - key: enable_mixed_source_conversation_continuity default: false - key: enable_mixed_source_development_telemetry default: false -- key: enable_mixed_source_manifest - default: false - key: enable_mixed_source_relevance_candidates - default: false + default: true - key: enable_multi_agent_orchestration default: false - key: enable_multi_model_endpoints diff --git a/docs/_data/features.yml b/docs/_data/features.yml index ad522e46e..8044fa953 100644 --- a/docs/_data/features.yml +++ b/docs/_data/features.yml @@ -73,8 +73,8 @@ grounded-document-search: name: "Grounded document search" area: "Chat and retrieval" audience: "End users" - summary: "Controls Azure AI Search-backed retrieval, cached search results, and mixed-source evidence reuse in chat." - why: "Use this when answers must be grounded in workspace documents and citations rather than only model memory. The mixed-source options are staged rollout controls, so enable them deliberately when follow-up questions should reuse previously authorized evidence." + summary: "Controls Azure AI Search-backed retrieval, cached search results, and how chat grounds answers in selected documents and spreadsheets." + why: "Use this when answers must be grounded in workspace documents and citations rather than only model memory. Grounding across documents and spreadsheets follows Enhanced Citations, which provides the spreadsheet engine; the remaining choice is whether chats that search without selected files also look for relevant spreadsheets, which adds a search and can add model calls." default_state: "Mixed" admin_tab: knowledge setting_keys: @@ -82,8 +82,6 @@ grounded-document-search: - enable_search_result_caching - enable_mixed_source_chat_search - enable_mixed_source_conversation_continuity - - enable_mixed_source_development_telemetry - - enable_mixed_source_manifest - enable_mixed_source_relevance_candidates reference: - title: "Chat interface reference" @@ -354,11 +352,12 @@ tabular-analysis: area: "Media and ingestion" audience: "End users" summary: "Runs structured CSV and spreadsheet analysis with durable planning, batching, checkpoints, and response controls." - why: "Use this when users need computed answers, grouped results, exports, or charts from spreadsheet data rather than prose guesses. The many rollout flags exist because large workbook analysis needs reliability controls for planning, retries, batching, and durable checkpoints." + why: "Use this when users need computed answers, grouped results, exports, or charts from spreadsheet data rather than prose guesses. The many rollout flags exist because large workbook analysis needs reliability controls for planning, retries, batching, and durable checkpoints. The Analyze All Documents target stays off and hidden because no chat or workflow screen offers it yet." default_state: "Mixed" admin_tab: agents-actions setting_keys: - enable_analysis_deliverable_contract_telemetry + - enable_mixed_source_analyze_all - enable_tabular_analyze_durable_preflight - enable_tabular_compact_response_protocol - enable_tabular_completion_driven_checkpointing @@ -380,10 +379,10 @@ document-compare: name: "Cross-format document compare" area: "Media and ingestion" audience: "End users" - summary: "Compares documents across supported formats, including staged one-to-many comparison behavior." - why: "Use this when users need revision review across PDFs, Office files, or other supported sources instead of manually reading each file. The one-to-many flag should be staged separately because it expands the amount of source material processed in one comparison." - default_state: "Disabled" - admin_tab: knowledge + summary: "Compares a Source with one or more Targets, including documents and spreadsheets in the same comparison." + why: "Use this when users need to review revisions or check the figures in a spreadsheet against a written document instead of reading each file by hand. Comparing documents with spreadsheets follows Enhanced Citations, which provides the spreadsheet engine; without it such a comparison is refused rather than treating the spreadsheet as text." + default_state: "Follows Enhanced Citations" + admin_tab: chat setting_keys: - enable_cross_format_compare - enable_cross_format_compare_one_to_many @@ -654,12 +653,13 @@ logging-and-diagnostics: name: "Logging and diagnostics" area: "Platform and operations" audience: "Operators" - summary: "Controls diagnostic logging, Application Insights global logging, and user-visible file processing logs." - why: "Use diagnostics when operators need to troubleshoot ingestion, settings, or runtime behavior. Keep verbose debug logging temporary because it can increase noise and operational cost." + summary: "Controls diagnostic logging, Application Insights global logging, mixed document and spreadsheet metrics, and user-visible file processing logs." + why: "Use diagnostics when operators need to troubleshoot ingestion, settings, or runtime behavior. Keep verbose debug logging and the mixed document and spreadsheet metrics temporary because they increase log volume and operational cost." default_state: "Mixed" admin_tab: operations setting_keys: - enable_appinsights_global_logging + - enable_mixed_source_development_telemetry - enable_debug_logging - enable_file_processing_logs reference: diff --git a/docs/admin/chat.md b/docs/admin/chat.md index 1766976a6..3774ddebc 100644 --- a/docs/admin/chat.md +++ b/docs/admin/chat.md @@ -254,15 +254,51 @@ appears. Chunk work normally reuses whichever model the user selected; pointing at a dedicated deployment keeps bulk row processing off the interactive model's quota. +**Documents and spreadsheets together.** With Enhanced Citations on, a CSV or Excel +file is indexed as a short description of its sheets and columns, and its rows are +handled by a spreadsheet engine that calculates over them instead of searching their +text. That engine only exists while Enhanced Citations is on, so the behaviors that +depend on it follow this switch rather than having switches of their own: + +- Chat and workflow Search use the files a user selected as the sources, even when the + Search Documents panel is closed. Each selected spreadsheet is analyzed on its own, + documents are searched as usual, and the answer says which sources could not be used. +- When a follow-up question with no new selection goes back to the earlier sources, + access is checked again, and a source that changed, was revoked, or was only partly + covered is worked on again instead of being answered from the earlier reply. +- Compare accepts documents and spreadsheets in the same comparison, with as many + Targets as the Comparison document limits allow. + +With Enhanced Citations off, spreadsheets are indexed row by row and searched as text, +the classic chat page uses selected documents only while its Search Documents panel is +on, and comparing a document with a spreadsheet is refused with a message that names +Enhanced Citations. If these behaviors ever need to be switched off during an incident, +set the `SIMPLECHAT_DISABLE_MIXED_SOURCE` environment variable to `true` on the app. +It applies to every settings read and changes nothing that is stored, so removing it +restores them. + +**Finding spreadsheets nobody selected.** A question about figures often lives in a +spreadsheet whose description does not rank among the first search results. +*Look for relevant spreadsheets when no files are selected* runs a second search aimed +at spreadsheet sheets and columns whenever a chat searches workspace documents without +specific files, and keeps up to six spreadsheets that fit the question. The extra +search adds a little time to those messages, and a matched spreadsheet that gets +analyzed adds model calls. Leave it on where people ask about data held in +spreadsheets; turn it off where those chats must stay as fast as possible. + #### Settings | Setting | What it does | Default | Notes | | --- | --- | --- | --- | -| Enable Enhanced Citations | Stores original files in Azure Storage so citations can open the source document, not just quoted text. | Off | `enable_enhanced_citations`; capability toggle | +| Enable Enhanced Citations | Stores original files in Azure Storage so citations can open the source document, not just quoted text. Also turns on spreadsheet analysis and the mixed document and spreadsheet behaviors below. | Off | `enable_enhanced_citations`; capability toggle | | Enhanced Citations mount path | Enables use of the configured Enhanced Citations mount path for document-view routing when Enhanced Citations is on; the saved value is forced off unless Enhanced Citations is enabled. | Off | `enable_enhanced_citations_mount`; no visible field in `admin_settings.html` | | Storage Account Authentication Type | Whether SimpleChat authenticates to the storage account with a connection string or a managed identity. | key | `office_docs_authentication_type` | | Storage Account Connection String | Credential used for connection string authentication. Stored write-only. | Empty | `office_docs_storage_account_url` | | Storage Account Blob Service Endpoint | Blob endpoint used for managed identity authentication. Stored write-only. | Empty | `office_docs_storage_account_blob_endpoint` | +| Look for relevant spreadsheets when no files are selected | When a chat searches workspace documents without specific files selected, runs a second search aimed at spreadsheet sheets and columns and keeps up to six spreadsheets that fit the question. | On | `enable_mixed_source_relevance_candidates`; shown while Enhanced Citations is on | +| Mixed document and spreadsheet Chat and Search | Uses selected files as the sources, analyzes each selected spreadsheet with the spreadsheet engine, and reports sources that could not be used. | Follows Enhanced Citations | `enable_mixed_source_chat_search`; derived, no control | +| Follow-up source continuity | Records which sources grounded an answer, so a follow-up rechecks access and works again on sources that changed or were only partly covered. | Follows Enhanced Citations | `enable_mixed_source_conversation_continuity`; derived, no control | +| Compare documents with spreadsheets | Lets one comparison combine documents and spreadsheets, including more than one Target. | Follows Enhanced Citations | `enable_cross_format_compare` and `enable_cross_format_compare_one_to_many`; derived, no control | | Maximum File Size for Tabular Preview (MB) | CSV and XLSX files above this size are not previewed, which protects the host from loading very large spreadsheets into memory. | 200 | `tabular_preview_max_blob_size_mb` | | Confirm very large row-level runs before starting | When a prompt includes an explicit large row count, users are asked to continue or narrow scope before the run starts. | On | `enable_tabular_durable_run_confirmation`; capability toggle | | Confirmation Row Threshold | Row count at or above which the confirmation is shown. | 500 | `tabular_durable_run_confirmation_threshold_rows` | diff --git a/docs/admin/operations.md b/docs/admin/operations.md index 833d1eed5..88ca0dae0 100644 --- a/docs/admin/operations.md +++ b/docs/admin/operations.md @@ -56,7 +56,18 @@ The Control Center Access section belongs to the Control Center tab. Use it with ### Application Insights {#application-insights-section} -The Application Insights section belongs to the Logging & Health tab. Use it with the adjacent settings in this group so related rollout, access, and operational choices stay aligned. +Global logging sends agent and orchestration events to Application Insights, so a +request can be traced across the components that handled it. Changing it requires an +application restart. + +*Record mixed document and spreadsheet metrics* adds `[MIXED_SOURCE_TELEMETRY]` +entries for the processing behind Chat, Search, Analyze and Compare over workspace +documents and spreadsheets: aggregate counts, timings and token totals showing how +many sources completed, were partial, failed, were skipped or were canceled, plus +authorization failures and background exports. The entries never carry prompts, +content, file names, document IDs or storage paths. Turn it on while checking how +those requests behave, for example after enabling Enhanced Citations, and off again +afterwards, because each request adds several log entries. ### Debug Logging {#debug-logging-section} @@ -82,6 +93,7 @@ The API Documentation section belongs to the Logging & Health tab. Use it with t | Enable /external/healthcheckz | Exposes the capability after required services, permissions, and rollout policy are ready. | Off | `enable_no_auth_external_healthcheck`; capability toggle | | Enable Swagger/OpenAPI Documentation (/swagger) | Exposes the capability after required services, permissions, and rollout policy are ready. | On | `enable_swagger`; capability toggle | | Enable Application Insights Global Logging | Sends global application, agent, and orchestration logging events to Application Insights. | Off | `enable_appinsights_global_logging`; capability toggle | +| Record mixed document and spreadsheet metrics | Logs aggregate counts, timings and token totals for Chat, Search, Analyze and Compare over workspace documents and spreadsheets, with no prompts, content, file names, document IDs or storage paths. | Off | `enable_mixed_source_development_telemetry`; capability toggle | | Enable Debug Logging | Captures verbose diagnostic logs for troubleshooting until disabled or the timer turns it off. | Off | `enable_debug_logging`; capability toggle | | Enable Time-based Auto Turnoff | Defines behavior for the related admin workflow; verify the affected feature after saving. | Off | `debug_logging_timer_enabled` | | Duration | Defines behavior for the related admin workflow; verify the affected feature after saving. | 1 | `debug_timer_value` | diff --git a/docs/admin_configuration.md b/docs/admin_configuration.md index 1ea8fc779..04140d5c8 100644 --- a/docs/admin_configuration.md +++ b/docs/admin_configuration.md @@ -197,11 +197,11 @@ For model endpoint identity details, see [Configure model endpoint identity]({{ -### Mixed-source rollout controls +### Mixed documents and spreadsheets -Mixed-source rollout is independently reversible. Keep `enable_mixed_source_manifest`, `enable_mixed_source_chat_search`, `enable_mixed_source_analyze`, `enable_cross_format_compare`, and `enable_mixed_source_conversation_continuity` off until the preceding stage is validated. The subordinate relevance, Analyze All, one-to-many Compare (`enable_cross_format_compare_one_to_many`), and development telemetry stages also default off. `enable_mixed_source_development_telemetry` records aggregate counts and latency only; it must never capture prompts, evidence, source identifiers, filenames, or storage paths. +Mixed document and spreadsheet behavior follows Enhanced Citations, because that is what provides the spreadsheet engine. `enable_mixed_source_chat_search`, `enable_mixed_source_conversation_continuity`, `enable_cross_format_compare`, and `enable_cross_format_compare_one_to_many` are rewritten to match `enable_enhanced_citations` on every settings load and save and have no admin control; set the `SIMPLECHAT_DISABLE_MIXED_SOURCE` environment variable to turn them off during an incident without changing stored settings. `enable_mixed_source_relevance_candidates` is the one administrator choice, under Chat > Citations > Enhanced, and defaults on. `enable_mixed_source_development_telemetry`, under Operations > Logging & Health > Application Insights, records aggregate counts and latency only and never captures prompts, evidence, source identifiers, filenames, or storage paths. `enable_mixed_source_analyze_all` gates an Analyze target of every document in scope that no chat or workflow screen can request, so it is not shown and stays off. The former `enable_mixed_source_analyze` and `enable_mixed_source_manifest` switches no longer exist and are removed from stored settings when they load. diff --git a/docs/explanation/features/CROSS_FORMAT_COMPARE.md b/docs/explanation/features/CROSS_FORMAT_COMPARE.md index 424170e00..433e62638 100644 --- a/docs/explanation/features/CROSS_FORMAT_COMPARE.md +++ b/docs/explanation/features/CROSS_FORMAT_COMPARE.md @@ -79,4 +79,16 @@ silently using preview rows or legacy file-producing execution. `functional_tests/test_orchestration_internal_analysis.py` covers complete per-target text after restart, partial and failed targets, full-report retention, cancellation, failed result guards, and zero managed uploads. The ordinary -standalone Compare entry point retains its existing default and return shape. \ No newline at end of file +standalone Compare entry point retains its existing default and return shape. + +## Settings Update (0.261.260) + +`enable_cross_format_compare` and `enable_cross_format_compare_one_to_many` are no +longer default-off stored switches. Both follow `enable_enhanced_citations` on every +settings load and save, because a mixed comparison needs the spreadsheet engine +Enhanced Citations provides; the Comparison document limits under Document Actions +still bound how many Targets a request may include. Without Enhanced Citations a +mixed request is still refused rather than comparing the spreadsheet as text, and the +message now names Enhanced Citations instead of a hidden rollout flag. +`SIMPLECHAT_DISABLE_MIXED_SOURCE` turns both off at read time. See +`docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. \ No newline at end of file diff --git a/docs/explanation/features/MIXED_SOURCE_ANALYZE.md b/docs/explanation/features/MIXED_SOURCE_ANALYZE.md index 84703b315..7b6f0fbb8 100644 --- a/docs/explanation/features/MIXED_SOURCE_ANALYZE.md +++ b/docs/explanation/features/MIXED_SOURCE_ANALYZE.md @@ -65,4 +65,8 @@ This phase does not implement cross-format Compare (#1059), persisted follow-up Version **0.250.070** completes the bounded backend contract for `enable_mixed_source_analyze_all` under [#1061](https://github.com/microsoft/simplechat/issues/1061). The ready document access index enumerates current candidates using the configured Analyze limit plus one, rejects over-limit catalogs without truncation, and sends every candidate through the existing authorized source manifest before execution. The flag remains default off and the workflow selector is not newly exposed pending staged production approval. -Combined Analyze now reduces only when at least one source succeeds. Terminal coverage, cancellation, generated outputs, citations, and finalization reauthorization are preserved across narrative and table branches. \ No newline at end of file +Combined Analyze now reduces only when at least one source succeeds. Terminal coverage, cancellation, generated outputs, citations, and finalization reauthorization are preserved across narrative and table branches. + +## Settings Update (0.261.260) + +`enable_mixed_source_analyze` has had no effect since 0.250.071 and is now removed from stored settings on load. `enable_mixed_source_analyze_all` keeps its default of off, is reset to off once by the 0.261.260 settings upgrade, and is no longer drawn in Admin Settings, because no chat or workflow screen offers an Analyze target of every document. See `docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. \ No newline at end of file diff --git a/docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md b/docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md index 2985d938c..7aafbab1b 100644 --- a/docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md +++ b/docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md @@ -165,4 +165,8 @@ Chat and Search remain relevance-bounded. Analyze and Compare behavior is otherw ## Phase 6 Hardening -Version **0.250.070** adds manifest-aligned terminal coverage, standard/streaming failure parity, and reference deduplication under [#1061](https://github.com/microsoft/simplechat/issues/1061). When a mixed request still has successful table evidence, a narrative retrieval or quota failure becomes an explicit per-source omission instead of aborting the available native branch. Chat and Search remain relevance bounded; no full catalog enumeration was added to either mode. \ No newline at end of file +Version **0.250.070** adds manifest-aligned terminal coverage, standard/streaming failure parity, and reference deduplication under [#1061](https://github.com/microsoft/simplechat/issues/1061). When a mixed request still has successful table evidence, a narrative retrieval or quota failure becomes an explicit per-source omission instead of aborting the available native branch. Chat and Search remain relevance bounded; no full catalog enumeration was added to either mode. + +## Settings Update (0.261.260) + +`enable_mixed_source_chat_search` is no longer a stored rollout switch. It follows `enable_enhanced_citations` on every settings load and save, because the native tabular branch needs the spreadsheet engine Enhanced Citations provides; without it, spreadsheets are indexed as text and the legacy path keeps serving them. The emergency rollback is the `SIMPLECHAT_DISABLE_MIXED_SOURCE` environment variable, applied at read time. `enable_mixed_source_relevance_candidates` is now an administrator switch under **Chat > Citations > Enhanced**, default on, and the Phase 1 shadow manifest (`enable_mixed_source_manifest`) is retired. See `docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. \ No newline at end of file diff --git a/docs/explanation/features/MIXED_SOURCE_CONVERSATION_CONTINUITY.md b/docs/explanation/features/MIXED_SOURCE_CONVERSATION_CONTINUITY.md index 79474d18d..237ec3cc6 100644 --- a/docs/explanation/features/MIXED_SOURCE_CONVERSATION_CONTINUITY.md +++ b/docs/explanation/features/MIXED_SOURCE_CONVERSATION_CONTINUITY.md @@ -39,4 +39,8 @@ Related version update: `application/single_app/config.py` moved from **0.250.06 ## Phase 6 Hardening -Version **0.250.070** preserves source version, terminal status, bounded coverage, role, and order through continuity normalization. A fresh manifest decision is now evaluated before history-only reuse, so revoked, changed, partial, failed, or truncated prior grounding forces native execution even when the history assessor would otherwise reuse an earlier answer. Chat-upload hints are filtered through fresh conversation ownership. \ No newline at end of file +Version **0.250.070** preserves source version, terminal status, bounded coverage, role, and order through continuity normalization. A fresh manifest decision is now evaluated before history-only reuse, so revoked, changed, partial, failed, or truncated prior grounding forces native execution even when the history assessor would otherwise reuse an earlier answer. Chat-upload hints are filtered through fresh conversation ownership. + +## Settings Update (0.261.260) + +`enable_mixed_source_conversation_continuity` now follows `enable_enhanced_citations` on every settings load and save, together with `enable_mixed_source_chat_search`, instead of being a default-off stored switch. `SIMPLECHAT_DISABLE_MIXED_SOURCE` turns both off at read time. See `docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. \ No newline at end of file diff --git a/docs/explanation/features/MIXED_SOURCE_HARDENING_EXTRACTION_AND_ROLLOUT.md b/docs/explanation/features/MIXED_SOURCE_HARDENING_EXTRACTION_AND_ROLLOUT.md index 9b02ae51d..c1bbdf0ce 100644 --- a/docs/explanation/features/MIXED_SOURCE_HARDENING_EXTRACTION_AND_ROLLOUT.md +++ b/docs/explanation/features/MIXED_SOURCE_HARDENING_EXTRACTION_AND_ROLLOUT.md @@ -110,3 +110,7 @@ The release validation also includes Python compilation, Pylance/editor diagnost - Analyze All remains a staged backend capability and is not default on or newly exposed in the selector. - Richer relational joins and persistent computed evidence caches remain future work. - Production telemetry evidence and explicit approval are still required before enabling mixed-source behavior by default. + +## Settings Update (0.261.260) + +The staged rollout above is complete. Mixed-source Chat and Search, conversation continuity, cross-format Compare and its one-to-many stage now follow `enable_enhanced_citations` on every settings load and save, because each needs the spreadsheet engine that Enhanced Citations provides; `SIMPLECHAT_DISABLE_MIXED_SOURCE` is the read-time rollback. `enable_mixed_source_relevance_candidates` (default on) and `enable_mixed_source_development_telemetry` (default off) are administrator switches with descriptions, `enable_mixed_source_analyze_all` stays off and hidden, and `enable_mixed_source_analyze` and `enable_mixed_source_manifest` are retired. See `docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. diff --git a/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md b/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md index d5d7aec74..9b062447b 100644 --- a/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md +++ b/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md @@ -112,4 +112,8 @@ Later phases can consume the shared partition and evidence contracts instead of ## Related Version Updates -- `application/single_app/config.py` was updated from **0.250.061** to **0.250.062** for #1056. \ No newline at end of file +- `application/single_app/config.py` was updated from **0.250.061** to **0.250.062** for #1056. + +## Settings Update (0.261.260) + +`enable_mixed_source_manifest` is retired. Analyze, Compare and mixed-source Chat and Search all resolve their manifests for real, so the shadow manifests it produced were discarded work. The key is removed from stored settings on load, and `_maybe_resolve_chat_source_manifest()` and the workflow shadow block are deleted. The manifest contracts in this document are unchanged. See `docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. \ No newline at end of file diff --git a/docs/explanation/features/V2_ADMIN_CHAT_SETTINGS.md b/docs/explanation/features/V2_ADMIN_CHAT_SETTINGS.md index 348747a0b..e693c93ed 100644 --- a/docs/explanation/features/V2_ADMIN_CHAT_SETTINGS.md +++ b/docs/explanation/features/V2_ADMIN_CHAT_SETTINGS.md @@ -143,8 +143,14 @@ claims there is something to edit. | --- | --- | | `enable_tabular_processing_plugin` | Derived from `enable_enhanced_citations` and rewritten on every read | | `enable_enhanced_citations_mount` | No control in either interface; forced off unless Enhanced Citations is on | -| `enable_mixed_source_chat_search` | Staged rollout flag with no administrator control | -| `enable_mixed_source_conversation_continuity` | Staged rollout flag gated behind the above | +| `enable_mixed_source_chat_search` | Derived from `enable_enhanced_citations` on every read and save (0.261.260) | +| `enable_mixed_source_conversation_continuity` | Derived the same way (0.261.260) | +| `enable_cross_format_compare` | Derived the same way (0.261.260) | +| `enable_cross_format_compare_one_to_many` | Derived the same way (0.261.260) | +| `enable_mixed_source_analyze_all` | Gates an Analyze target no chat or workflow screen can request (0.261.260) | + +Until 0.261.260 the two mixed-source entries were described as staged rollout flags +with no administrator control. See `MIXED_SOURCE_ADMIN_SETTINGS_FIX.md`. The list is served on the settings GET as `suppressed_capabilities` and honoured by `buildCapabilityIndex`. diff --git a/docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md b/docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md new file mode 100644 index 000000000..e9650ffda --- /dev/null +++ b/docs/explanation/fixes/MIXED_SOURCE_ADMIN_SETTINGS_FIX.md @@ -0,0 +1,145 @@ +# Mixed-Source Admin Settings Fix + +Fixed/Implemented in version: **0.261.260** + +Related config.py update: `VERSION = "0.261.260"` + +## Header Information + +### Issue description + +Admin Settings in the V2 interface showed five switches under **Knowledge > Web & +Research > Deep Research** with no description, only a label generated from each +key name: + +| Switch shown | What it actually did | +| --- | --- | +| Mixed source analyze (`enable_mixed_source_analyze`) | Nothing. The gate was removed in 0.250.071, when combined Analyze began routing mixed narrative and tabular selections natively. The key survived only in stored settings documents. | +| Mixed source analyze all (`enable_mixed_source_analyze_all`) | Allowed an Analyze target of every document in scope, which no chat or workflow screen can request. | +| Mixed source manifest (`enable_mixed_source_manifest`) | Phase 1 shadow mode: resolved a source manifest in the background, discarded it, and logged on failure. Every mixed-source path now resolves its manifest for real. | +| Mixed source relevance candidates (`enable_mixed_source_relevance_candidates`) | Nothing on its own. It extends `enable_mixed_source_chat_search`, which had no control in either admin interface. | +| Mixed source development telemetry (`enable_mixed_source_development_telemetry`) | Aggregate `[MIXED_SOURCE_TELEMETRY]` counts and timings in Application Insights. | + +The behaviors that mattered most had no control at all. `enable_mixed_source_chat_search` +and `enable_mixed_source_conversation_continuity` were suppressed from the V2 page and +absent from the classic page, and `enable_cross_format_compare` and +`enable_cross_format_compare_one_to_many` were filed under "Other capabilities". All of +them defaulted off, so: + +- The classic chat page ignored selected documents whenever its Search Documents panel + was closed. +- Workflow Search did not run selected spreadsheets through the spreadsheet engine. +- Comparing a document with a spreadsheet failed with "Mixed narrative and tabular + Compare is temporarily unavailable while cross-format Compare is disabled." + +### Root cause analysis + +1. **Placement.** None of the keys was declared in `admin_settings_fields.py`, so the V2 + page drew them with its `enable_*` fallback scan, which files a key under the first + section sharing the most word stems. Every `enable_mixed_source_*` key matched only + "source" in `source-review-section`, which is Deep Research. The cross-format pair + matched nothing. +2. **Stale keys.** Commit `a4834912d` removed the defaults for `enable_mixed_source_analyze` + and `enable_mixed_source_analyze_all`, but `deep_merge_dicts()` never deletes keys, so + deployments that had loaded settings earlier kept both in Cosmos DB, and the scan kept + drawing them. +3. **Staged rollout never finished.** The mixed-source phases (#1055 to #1061) shipped + behind default-off switches pending production approval. The switches that gate the + user-facing behavior were never exposed, so nobody could approve them from the + product. + +### Version implemented + +0.261.260 + +## Technical Details + +### Decisions + +Every mixed-source tabular step needs `is_tabular_processing_enabled()`, which is +`enable_enhanced_citations`. With Enhanced Citations on, a spreadsheet is indexed as one +schema summary chunk and has to go through the spreadsheet engine; with it off, rows are +indexed as text chunks. Forcing the mixed-source path on everywhere would have turned +spreadsheets in deployments without Enhanced Citations from "searched as text" into +"unavailable", because Phase 6 forbids a table falling back to narrative processing. + +| Key | Now | +| --- | --- | +| `enable_mixed_source_chat_search` | Derived from `enable_enhanced_citations` on every load and save; suppressed in V2 | +| `enable_mixed_source_conversation_continuity` | Derived the same way; suppressed | +| `enable_cross_format_compare` | Derived the same way; suppressed | +| `enable_cross_format_compare_one_to_many` | Derived the same way; suppressed. The Comparison document limits still bound Targets. | +| `enable_mixed_source_relevance_candidates` | Administrator choice under **Chat > Citations > Enhanced**, shown while Enhanced Citations is on. Default **on**; a one-time upgrade turns it on for existing deployments. | +| `enable_mixed_source_development_telemetry` | Administrator choice under **Operations > Logging & Health > Application Insights**, default off | +| `enable_mixed_source_analyze_all` | Default restored to off and suppressed in V2. The one-time upgrade resets it to off. | +| `enable_mixed_source_analyze` | Retired; removed from stored settings on load | +| `enable_mixed_source_manifest` | Retired; removed on load, and the shadow-resolution code is deleted | + +The derived behaviors have one rollback path: the `SIMPLECHAT_DISABLE_MIXED_SOURCE` +environment variable, applied to every settings read in `_format_result` and never +persisted. This mirrors `SIMPLECHAT_DISABLE_TABULAR_PARITY_DURABLE_PREFLIGHT`. + +`enable_appinsights_global_logging` is now declared in `application-insights-section` +too. The scan had filed it under Debug Logging on the word "logging", which would have +left the telemetry switch alone in an otherwise empty Application Insights section. + +### Files modified + +| File | Change | +| --- | --- | +| `application/single_app/functions_settings.py` | `MIXED_SOURCE_DERIVED_SETTING_KEYS`, `normalize_mixed_source_derived_settings()`, `normalize_mixed_source_settings_upgrade()`, `_apply_mixed_source_env_kill_switch()`; two keys added to `RETIRED_SETTING_KEYS`; defaults updated; `is_mixed_source_manifest_enabled()` removed | +| `application/single_app/route_backend_chats.py` | `_maybe_resolve_chat_source_manifest()` and its two call sites removed | +| `application/single_app/functions_workflow_runner.py` | Shadow manifest block removed from `_maybe_execute_tabular_document_action()`; Compare limitation message names Enhanced Citations | +| `application/single_app/admin_settings_fields.py` | Relevance candidates declared under Enhanced Citations; `application-insights-section` declared; Enhanced Citations help text names spreadsheet analysis; suppression reasons updated and three keys added | +| `application/single_app/templates/admin/_panes/citation.html` | "Spreadsheets in Chat" card with the relevance switch | +| `application/single_app/templates/admin/_panes/logging.html` | Metrics switch under Application Insights | +| `application/single_app/route_frontend_admin_settings.py` | Classic save persists both switches | + +### One-time upgrade + +`mixed_source_settings_version` (default `0`) records whether the upgrade ran. On the +first load at version 1 it turns relevance candidates on and Analyze All off, then +stores the marker, so an administrator who turns relevance candidates off later keeps +that choice. It runs on load only, never on save. + +### Impact + +- Deployments with Enhanced Citations on start using mixed-source Chat and Search, + follow-up continuity and cross-format Compare on the first settings load after + upgrading. +- The first load rewrites the settings document once. Saved orchestration runs bind to + a fingerprint of the full settings, so a run paused across the upgrade may report that + its execution context changed, exactly as after any administrator save. +- With Enhanced Citations off, behavior is unchanged, apart from the actionable Compare + message. The classic chat page still uses selected documents only while its Search + Documents panel is on; the V2 chat already sends them whenever they are attached. +- The flag-off branches remain in code as the kill switch path. + +## Validation + +### Tests + +| Test | Covers | +| --- | --- | +| `functional_tests/test_mixed_source_settings_derivation.py` | Derivation from Enhanced Citations, read-time kill switch, retired keys purged and unseeded, one-time upgrade and later choices, admin declarations in both interfaces, classic save, Compare message | +| `functional_tests/test_v2_admin_capability_placement.py` | New relocations and suppressions; no mixed-source key left to the fallback scan | +| `ui_tests/test_v2_admin_mixed_source_settings.py` | Built V2 page with the real schema: nothing mixed-source under Deep Research, both switches described in their sections, the spreadsheet switch hidden without Enhanced Citations, saves through the field normalizer; classic panes submit both switches | +| Updated suites | `test_mixed_source_chat_search_consistency.py`, `test_mixed_source_conversation_continuity.py`, `test_cross_format_compare_workflow.py`, `test_tabular_document_actions_workflow.py`, `test_orchestration_single_contract.py`, `test_docs_app_surface_coverage.py`, and the settings writer tests that stub normalizers | + +### Before and after + +| | Before | After | +| --- | --- | --- | +| Deep Research section | Five unlabelled mixed-source switches | Deep Research settings only | +| Relevance candidates | Shown, no effect | Described under Enhanced Citations, effective, default on | +| Telemetry | Under Deep Research | Under Application Insights with a privacy statement | +| PDF compared with XLSX | Fails while the hidden switch is off | Works with Enhanced Citations; otherwise refused with a message naming it | +| Classic chat, panel closed, Enhanced Citations on | Selected documents ignored | Selected documents used | + +## Related + +- `docs/admin/chat.md` (Enhanced Citations) and `docs/admin/operations.md` (Application Insights) +- `docs/explanation/fixes/MIXED_SOURCE_ANALYZE_GATING_REMOVAL_FIX.md` +- `docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md` +- `docs/explanation/features/CROSS_FORMAT_COMPARE.md` +- `docs/explanation/features/V2_ADMIN_CHAT_SETTINGS.md` diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index a3ca6cf0d..c6d109a1a 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -6,6 +6,7 @@ order: 120 category: Version History --- +- [Mixed-Source Admin Settings Fix](MIXED_SOURCE_ADMIN_SETTINGS_FIX.md) - [Deleted Messages Shown as Masked Fix](SOFT_DELETED_MESSAGES_SHOWN_AS_MASKED_FIX.md) - [Shared Conversation Diagram Editing Fix](SHARED_CONVERSATION_DIAGRAM_EDITING_FIX.md) - [Chat Document Search File Name and Divider Artifact Fix](CHAT_DOCUMENT_SEARCH_FILENAME_AND_DIVIDER_FIX.md) diff --git a/docs/reference/application-workflows.md b/docs/reference/application-workflows.md index 903455c38..77ba7c295 100644 --- a/docs/reference/application-workflows.md +++ b/docs/reference/application-workflows.md @@ -87,8 +87,8 @@ Typical chunk metadata includes document identity, filename, workspace scope, se ## Mixed-Source Document Actions -When mixed-source flags are enabled, Chat and workflow Search remain relevance bounded. Analyze and Compare resolve an ordered authorized manifest, dispatch narrative documents to bounded window analysis and CSV/Excel documents to native tabular tools, and combine only bounded evidence summaries. +When Enhanced Citations is enabled, Chat and workflow Search remain relevance bounded. Analyze and Compare resolve an ordered authorized manifest, dispatch narrative documents to bounded window analysis and CSV/Excel documents to native tabular tools, and combine only bounded evidence summaries. Without Enhanced Citations there is no spreadsheet engine, so a Compare that mixes a document with a spreadsheet is refused with a message naming Enhanced Citations rather than comparing the spreadsheet as text. Every selected or planned source ends with `completed`, `partial`, `failed`, or `skipped` plus a bounded reason. Analyze reduces only when at least one source succeeds. Compare requires a prepared Source and can continue past a failed Target. Cancellation stops active branches and prevents later reduction, citation/artifact publication, and assistant response persistence. -The staged Analyze All backend enumerates current documents through the ready document access index and rejects catalogs above the configured workflow Analyze limit. Every enumerated ID is reauthorized before execution. This stage remains default off and is not newly exposed in the workflow selector in version **0.250.070**. +The staged Analyze All backend enumerates current documents through the ready document access index and rejects catalogs above the configured workflow Analyze limit. Every enumerated ID is reauthorized before execution. It is gated by `enable_mixed_source_analyze_all`, which stays off and is not shown in Admin Settings, because no chat or workflow screen offers an Analyze target of every document. diff --git a/functional_tests/test_analyze_native_saved_integration.py b/functional_tests/test_analyze_native_saved_integration.py index 3fcc9c966..8544af72d 100644 --- a/functional_tests/test_analyze_native_saved_integration.py +++ b/functional_tests/test_analyze_native_saved_integration.py @@ -317,7 +317,7 @@ async def generate(**kwargs): plan_tabular_request=lambda *args, **kwargs: {}, run_tabular_analysis_with_thought_tracking=foreground, )) runner = runner_namespace( - asyncio=asyncio, is_mixed_source_manifest_enabled=lambda settings: False, + asyncio=asyncio, is_tabular_processing_enabled=lambda settings: True, get_plugin_logger=lambda: SimpleNamespace(get_invocations_for_conversation=lambda *args, **kwargs: []), classify_tabular_parity_request=lambda text: {}, diff --git a/functional_tests/test_app_settings_store_consistency.py b/functional_tests/test_app_settings_store_consistency.py index cd1a40cf3..81a1314e1 100644 --- a/functional_tests/test_app_settings_store_consistency.py +++ b/functional_tests/test_app_settings_store_consistency.py @@ -317,7 +317,7 @@ def load_update_settings(store): "normalize_group_workflow_assignment_settings", "normalize_agents_page_promoted_popular_settings", "normalize_document_access_index_required_settings", "normalize_inbound_mcp_settings", "normalize_public_workspace_display_settings", "normalize_key_vault_reminder_settings", - "normalize_model_endpoint_identity_header_settings", + "normalize_model_endpoint_identity_header_settings", "normalize_mixed_source_derived_settings", ): namespace[name] = lambda settings: None nodes = [ @@ -380,6 +380,7 @@ def load_get_settings(store): "SettingsUnavailableError": SettingsUnavailableError, "log_event": lambda *_args, **_kwargs: None, "_apply_tabular_parity_env_kill_switch": lambda settings: settings, + "_apply_mixed_source_env_kill_switch": lambda settings: settings, "attach_public_workspace_label_context": lambda settings: settings, "normalize_document_intelligence_pdf_image_extraction_mode": lambda mode: mode, "is_tabular_processing_enabled": lambda settings: False, diff --git a/functional_tests/test_content_screening_settings_api.py b/functional_tests/test_content_screening_settings_api.py index 0faba8c15..43e2c11d5 100644 --- a/functional_tests/test_content_screening_settings_api.py +++ b/functional_tests/test_content_screening_settings_api.py @@ -111,6 +111,7 @@ def settings_functions(**overrides): "normalize_public_workspace_display_settings", "normalize_key_vault_reminder_settings", "normalize_model_endpoint_identity_header_settings", "normalize_retired_orchestration_settings", + "normalize_mixed_source_derived_settings", ): namespace[name] = lambda _settings: None namespace.update(overrides) diff --git a/functional_tests/test_cross_format_compare_workflow.py b/functional_tests/test_cross_format_compare_workflow.py index 8dd286225..60ef54a5b 100644 --- a/functional_tests/test_cross_format_compare_workflow.py +++ b/functional_tests/test_cross_format_compare_workflow.py @@ -2,8 +2,8 @@ # test_cross_format_compare_workflow.py """ Functional test for Phase 4 cross-format Compare. -Version: 0.250.067 -Implemented in: 0.250.067 +Version: 0.261.260 +Implemented in: 0.250.067; derived from Enhanced Citations in 0.261.260 This test ensures #1059 retains one Source and ordered Targets, uses bounded native evidence, and preserves failed Targets during pairwise reduction. @@ -107,16 +107,29 @@ def test_cross_format_coordinator_uses_native_partitions_and_rollout_guards(): assert "'computed tabular facts'" in helper_source assert "'narrative document analysis'" in helper_source assert 'is_cross_format_compare_one_to_many_enabled(settings)' in helper_source - assert 'Mixed narrative and tabular Compare is temporarily unavailable while cross-format Compare is disabled.' in source + assert 'Comparing a document with a spreadsheet requires spreadsheet processing' in source + assert 'temporarily unavailable while cross-format Compare is disabled' not in source -def test_phase_4_flags_default_off_and_all_runner_paths_use_them(): - """Model and agent Compare retain flag-off rollback and staged one-to-many rollout.""" +def _derived_mixed_source_keys(settings_source): + """Read MIXED_SOURCE_DERIVED_SETTING_KEYS without importing configuration.""" + for node in ast.parse(settings_source).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == 'MIXED_SOURCE_DERIVED_SETTING_KEYS' + for target in node.targets + ): + return ast.literal_eval(node.value) + raise AssertionError('MIXED_SOURCE_DERIVED_SETTING_KEYS is missing from functions_settings.py') + + +def test_phase_4_flags_follow_enhanced_citations_and_all_runner_paths_use_them(): + """Compare follows Enhanced Citations; model and agent paths keep the rollback check.""" settings_source = SETTINGS.read_text(encoding='utf-8') workflow_source = WORKFLOW.read_text(encoding='utf-8') - assert "'enable_cross_format_compare': False" in settings_source - assert "'enable_cross_format_compare_one_to_many': False" in settings_source + derived_keys = _derived_mixed_source_keys(settings_source) + assert 'enable_cross_format_compare' in derived_keys + assert 'enable_cross_format_compare_one_to_many' in derived_keys assert 'def is_cross_format_compare_enabled(settings):' in settings_source assert 'def is_cross_format_compare_one_to_many_enabled(settings):' in settings_source assert workflow_source.count('mixed_comparison_enabled = is_cross_format_compare_enabled(settings)') == 2 @@ -126,5 +139,5 @@ def test_phase_4_flags_default_off_and_all_runner_paths_use_them(): if __name__ == '__main__': test_pairwise_reducer_preserves_target_order_and_partial_failure() test_cross_format_coordinator_uses_native_partitions_and_rollout_guards() - test_phase_4_flags_default_off_and_all_runner_paths_use_them() + test_phase_4_flags_follow_enhanced_citations_and_all_runner_paths_use_them() print('Phase 4 cross-format Compare tests passed.') \ No newline at end of file diff --git a/functional_tests/test_docs_app_surface_coverage.py b/functional_tests/test_docs_app_surface_coverage.py index 343b0e707..d628aedcc 100644 --- a/functional_tests/test_docs_app_surface_coverage.py +++ b/functional_tests/test_docs_app_surface_coverage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Functional test for documentation coverage of the application surface. -Version: 0.260.020 +Version: 0.261.260 Implemented in: 0.250.230 This test ensures that every user-facing capability of SimpleChat has @@ -56,7 +56,6 @@ CAPABILITY_EXEMPTIONS = { "enable_analysis_deliverable_contract_telemetry": "Internal telemetry switch for deliverable contract diagnostics.", "enable_tabular_parity_contract_telemetry": "Internal telemetry switch for tabular parity diagnostics.", - "enable_mixed_source_development_telemetry": "Internal development telemetry switch.", "enable_dai_debug": "Internal Document Intelligence debug switch.", "enable_tabular_generation_balanced_batches": "Internal tabular batching tuning flag.", "enable_tabular_completion_driven_checkpointing": "Internal tabular checkpointing tuning flag.", @@ -76,8 +75,6 @@ "enable_document_access_index_shadow_validation": "Internal Cosmos access-index rollout flag.", "enable_document_access_index_write_through": "Internal Cosmos access-index rollout flag.", "enable_startup_document_access_index_backfill": "Internal Cosmos access-index rollout flag.", - "enable_mixed_source_manifest": "Internal mixed-source rollout flag.", - "enable_mixed_source_relevance_candidates": "Internal mixed-source rollout flag.", "enable_default_embedding_model_plugin": "Internal plugin registration default.", } diff --git a/functional_tests/test_image_chat_route_integration.py b/functional_tests/test_image_chat_route_integration.py index 5ee04311f..79e7d6391 100644 --- a/functional_tests/test_image_chat_route_integration.py +++ b/functional_tests/test_image_chat_route_integration.py @@ -224,7 +224,6 @@ def build_chat_namespace(self): "debug_print": Mock(), "log_event": self.logs, "is_mixed_source_chat_search_enabled": Mock(return_value=False), - "is_mixed_source_manifest_enabled": Mock(return_value=False), "is_source_review_enabled_for_user": Mock(return_value=False), "is_url_access_enabled_for_user": Mock(return_value=False), "extract_urls_from_text": Mock(return_value=[]), @@ -257,7 +256,7 @@ def build_chat_namespace(self): load_route_functions("route_backend_chats.py", ( "_metadata_item_count", "_safe_metadata_int", "_normalize_capability_action", "_normalize_conversation_task_document_ids", "_normalize_chat_document_context_contract", - "_maybe_resolve_chat_source_manifest", "_source_review_metadata_used", "_deep_research_query_count", + "_source_review_metadata_used", "_deep_research_query_count", "_build_capability_usage_metadata", "_conversation_title_is_default", "_set_initial_conversation_title", "_resolve_pending_generated_file_format", "_resolve_generated_file_guidance_format", "_resolve_canonical_chat_agent", "_get_chat_agent_selection_name", "_has_chat_agent_selection", diff --git a/functional_tests/test_mixed_source_chat_search_consistency.py b/functional_tests/test_mixed_source_chat_search_consistency.py index 535831a15..e86571807 100644 --- a/functional_tests/test_mixed_source_chat_search_consistency.py +++ b/functional_tests/test_mixed_source_chat_search_consistency.py @@ -2,8 +2,9 @@ # test_mixed_source_chat_search_consistency.py """ Functional test for mixed-source Chat and Search consistency. -Version: 0.260.025 -Implemented in: 0.250.064; additive tabular evidence gating updated in 0.260.025 +Version: 0.261.260 +Implemented in: 0.250.064; additive tabular evidence gating updated in 0.260.025; +derived from Enhanced Citations in 0.261.260 This test ensures Phase 2 of #1057 consumes the Phase 1 #1056 contracts for standard and streaming Chat plus workflow Search without implementing later @@ -772,14 +773,29 @@ def test_replay_collaboration_foundry_and_chat_path_contracts(): assert "if hybrid_search_enabled or history_grounded_search_used" not in route_source -def test_flag_defaults_off_and_diagnostics_are_privacy_safe(): +def _derived_mixed_source_keys(settings_source): + """Read MIXED_SOURCE_DERIVED_SETTING_KEYS without importing configuration.""" + for node in ast.parse(settings_source).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "MIXED_SOURCE_DERIVED_SETTING_KEYS" + for target in node.targets + ): + return ast.literal_eval(node.value) + raise AssertionError("MIXED_SOURCE_DERIVED_SETTING_KEYS is missing from functions_settings.py") + + +def test_flag_follows_enhanced_citations_and_diagnostics_are_privacy_safe(): settings_source = _read(SETTINGS_PATH) - assert "'enable_mixed_source_chat_search': False" in settings_source - assert "'enable_mixed_source_relevance_candidates': False" in settings_source - assert "enable_mixed_source_manifest" not in _read(ROUTE_PATH)[ - _read(ROUTE_PATH).find("def _normalize_chat_document_context_contract"): - _read(ROUTE_PATH).find("def _resolve_chat_mixed_source_manifest") - ] + route_source = _read(ROUTE_PATH) + # Chat and Search behavior follows Enhanced Citations on every load and save rather + # than being a stored rollout switch; relevance candidates are an admin choice, on. + assert "enable_mixed_source_chat_search" in _derived_mixed_source_keys(settings_source) + assert "normalize_mixed_source_derived_settings(merged)" in settings_source + assert "normalize_mixed_source_derived_settings(settings_item)" in settings_source + assert "'enable_mixed_source_relevance_candidates': True" in settings_source + # The Phase 1 shadow manifest is retired; Chat resolves its manifest for real. + assert "enable_mixed_source_manifest" not in route_source + assert "_maybe_resolve_chat_source_manifest" not in route_source captured_events = [] original_log_event = orchestration.log_event diff --git a/functional_tests/test_mixed_source_conversation_continuity.py b/functional_tests/test_mixed_source_conversation_continuity.py index 5f3e57d4e..31b474576 100644 --- a/functional_tests/test_mixed_source_conversation_continuity.py +++ b/functional_tests/test_mixed_source_conversation_continuity.py @@ -2,8 +2,8 @@ # test_mixed_source_conversation_continuity.py """ Functional test for Phase 5 mixed-source conversation continuity. -Version: 0.250.107 -Implemented in: 0.250.068; updated in 0.250.107 +Version: 0.261.260 +Implemented in: 0.250.068; updated in 0.250.107; derived from Enhanced Citations in 0.261.260 This test ensures #1060 preserves compact source continuity only as a reauthorization hint for #1055 and prerequisite phases #1056, #1057, #1058, @@ -103,14 +103,28 @@ def test_explicit_selection_overrides_and_history_requires_reauthorization(): print('PASS: selection precedence and reauthorization') +def _derived_mixed_source_keys(settings_source): + """Read MIXED_SOURCE_DERIVED_SETTING_KEYS without importing configuration.""" + for node in ast.parse(settings_source).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == 'MIXED_SOURCE_DERIVED_SETTING_KEYS' + for target in node.targets + ): + return ast.literal_eval(node.value) + raise AssertionError('MIXED_SOURCE_DERIVED_SETTING_KEYS is missing from functions_settings.py') + + def test_flag_and_standard_streaming_wiring_are_present(): - """Both Chat paths retain flag-off rollback and only use fresh manifest results.""" + """Both Chat paths retain the derived-flag rollback and only use fresh manifest results.""" print('Testing flag and Chat parity wiring...') settings_source = _read_file(SETTINGS_FILE) route_source = _read_file(ROUTE_FILE) metadata_source = _read_file(METADATA_FILE) config_source = _read_file(CONFIG_FILE) - assert "'enable_mixed_source_conversation_continuity': False" in settings_source + # Continuity follows Enhanced Citations with Chat and Search, so it is not a stored switch. + derived_keys = _derived_mixed_source_keys(settings_source) + assert 'enable_mixed_source_conversation_continuity' in derived_keys + assert 'enable_mixed_source_chat_search' in derived_keys assert 'def is_mixed_source_conversation_continuity_enabled(settings):' in settings_source assert route_source.count('is_mixed_source_conversation_continuity_enabled(settings)') == 5 assert route_source.count('_build_reauthorized_continuity_decision(') == 4 diff --git a/functional_tests/test_mixed_source_settings_derivation.py b/functional_tests/test_mixed_source_settings_derivation.py new file mode 100644 index 000000000..1683a3f8a --- /dev/null +++ b/functional_tests/test_mixed_source_settings_derivation.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +# test_mixed_source_settings_derivation.py +""" +Functional test for the mixed-source admin settings clean-up. +Version: 0.261.260 +Implemented in: 0.261.260 + +The V2 admin page drew five mixed-source switches under Knowledge > Web & Research > +Deep Research, labelled only with their key names, because nothing declared them and the +fallback scan matched the word "source". Two of them did nothing: one had been retired in +0.250.071 and survived only in stored settings, and one depended on a hidden switch. + +This test ensures that: + +- Mixed-source Chat and Search, follow-up continuity and cross-format Compare follow + Enhanced Citations on every settings load and save, because that is when the + spreadsheet engine they need exists. +- SIMPLECHAT_DISABLE_MIXED_SOURCE turns them off at read time without changing what is + stored. +- The retired switches are purged from stored settings. +- A one-time upgrade turns relevance candidates on and Analyze All off, and an + administrator's later choice survives every subsequent load. +- The two real choices are declared with descriptions where they belong, in both admin + interfaces, and the classic save persists them. +""" + +import ast +import copy +import os +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent)) + +from test_support.app_stubs import import_app_module +from test_support.versioning import assert_app_version_at_least + + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT / "application" / "single_app" +SETTINGS_FILE = APP_ROOT / "functions_settings.py" +WORKFLOW_RUNNER_FILE = APP_ROOT / "functions_workflow_runner.py" +ADMIN_ROUTE_FILE = APP_ROOT / "route_frontend_admin_settings.py" +PANES_DIR = APP_ROOT / "templates" / "admin" / "_panes" + +IMPLEMENTED_VERSION = "0.261.260" +KILL_SWITCH = "SIMPLECHAT_DISABLE_MIXED_SOURCE" +DERIVED_KEYS = ( + "enable_mixed_source_chat_search", + "enable_mixed_source_conversation_continuity", + "enable_cross_format_compare", + "enable_cross_format_compare_one_to_many", +) +RETIRED_KEYS = ("enable_mixed_source_analyze", "enable_mixed_source_manifest") +SETTINGS_NAMES = { + "is_tabular_processing_enabled", + "_env_flag_enabled", + "is_mixed_source_chat_search_enabled", + "is_mixed_source_conversation_continuity_enabled", + "is_cross_format_compare_enabled", + "is_cross_format_compare_one_to_many_enabled", + "is_mixed_source_relevance_candidates_enabled", + "normalize_mixed_source_derived_settings", + "normalize_mixed_source_settings_upgrade", + "_apply_mixed_source_env_kill_switch", + "normalize_retired_orchestration_settings", + "deep_merge_dicts", + "MIXED_SOURCE_DERIVED_SETTING_KEYS", + "MIXED_SOURCE_KILL_SWITCH_ENV", + "MIXED_SOURCE_SETTINGS_VERSION_KEY", + "MIXED_SOURCE_SETTINGS_VERSION", + "RETIRED_SETTING_KEYS", +} + + +def _settings_source(): + return SETTINGS_FILE.read_text(encoding="utf-8") + + +def load_settings_functions(): + """Load the production helpers without importing config or Azure clients.""" + tree = ast.parse(_settings_source(), filename=str(SETTINGS_FILE)) + selected = [] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name in SETTINGS_NAMES: + selected.append(node) + elif isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id in SETTINGS_NAMES + for target in node.targets + ): + selected.append(node) + namespace = {"os": os} + exec(compile(ast.Module(body=selected, type_ignores=[]), str(SETTINGS_FILE), "exec"), namespace) + missing = sorted(name for name in SETTINGS_NAMES if name not in namespace) + assert not missing, f"functions_settings.py no longer defines: {missing}" + return namespace + + +def _get_settings_node(): + tree = ast.parse(_settings_source(), filename=str(SETTINGS_FILE)) + return next( + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "get_settings" + ) + + +def read_default_settings(keys): + """Return the literal defaults get_settings() ships for the named keys.""" + get_settings = _get_settings_node() + defaults_node = next( + statement.value for statement in get_settings.body + if isinstance(statement, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "default_settings" for target in statement.targets) + ) + found = {} + for key_node, value_node in zip(defaults_node.keys, defaults_node.values): + if isinstance(key_node, ast.Constant) and key_node.value in keys: + found[key_node.value] = ast.literal_eval(value_node) + return found + + +def all_default_keys(): + get_settings = _get_settings_node() + defaults_node = next( + statement.value for statement in get_settings.body + if isinstance(statement, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "default_settings" for target in statement.targets) + ) + return { + key_node.value for key_node in defaults_node.keys + if isinstance(key_node, ast.Constant) + } + + +def _source_segment(function_name, *, nested_in=None): + source = _settings_source() + tree = ast.parse(source, filename=str(SETTINGS_FILE)) + scope = tree + if nested_in: + scope = next( + node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == nested_in + ) + node = next( + item for item in ast.walk(scope) + if isinstance(item, ast.FunctionDef) and item.name == function_name + ) + return ast.get_source_segment(source, node) or "" + + +def simulate_settings_load(functions, stored, defaults): + """Replay the mixed-source steps get_settings() applies to a stored document.""" + persisted = copy.deepcopy(stored) + functions["deep_merge_dicts"](defaults, persisted) + functions["normalize_retired_orchestration_settings"](persisted) + functions["normalize_mixed_source_derived_settings"](persisted) + functions["normalize_mixed_source_settings_upgrade"](persisted) + returned = functions["_apply_mixed_source_env_kill_switch"](copy.deepcopy(persisted)) + return persisted, returned + + +def _defaults_for_simulation(): + keys = set(DERIVED_KEYS) | { + "enable_enhanced_citations", + "enable_mixed_source_relevance_candidates", + "enable_mixed_source_analyze_all", + "enable_mixed_source_development_telemetry", + "mixed_source_settings_version", + } + defaults = read_default_settings(keys) + assert set(defaults) == keys, f"Missing defaults: {sorted(keys - set(defaults))}" + return defaults + + +def test_derived_behaviors_follow_enhanced_citations(): + """Each derived behavior is on exactly when spreadsheet processing exists.""" + print("Testing that the derived mixed-source behaviors follow Enhanced Citations...") + assert_app_version_at_least(IMPLEMENTED_VERSION) + functions = load_settings_functions() + derive = functions["normalize_mixed_source_derived_settings"] + + assert tuple(functions["MIXED_SOURCE_DERIVED_SETTING_KEYS"]) == DERIVED_KEYS + + stored_off = {"enable_enhanced_citations": True, **{key: False for key in DERIVED_KEYS}} + assert derive(stored_off) is True + assert all(stored_off[key] is True for key in DERIVED_KEYS) + assert derive(stored_off) is False, "A second pass must not report a change" + + stored_on = {"enable_enhanced_citations": False, **{key: True for key in DERIVED_KEYS}} + assert derive(stored_on) is True + assert all(stored_on[key] is False for key in DERIVED_KEYS) + + # A string left by an old save is corrected to a real boolean. + stringly = {"enable_enhanced_citations": True, "enable_cross_format_compare": "true"} + derive(stringly) + assert stringly["enable_cross_format_compare"] is True + + # The runtime gates read the derived values unchanged. + assert functions["is_mixed_source_chat_search_enabled"](stored_off) is True + assert functions["is_cross_format_compare_one_to_many_enabled"](stored_off) is True + assert functions["is_mixed_source_chat_search_enabled"](stored_on) is False + print(" Derived behaviors follow Enhanced Citations.") + return True + + +def test_subordinate_switches_need_their_parent(): + """Relevance candidates and continuity have no effect without Chat and Search.""" + print("\nTesting subordinate gating...") + functions = load_settings_functions() + without_chat = { + "enable_mixed_source_chat_search": False, + "enable_mixed_source_relevance_candidates": True, + "enable_mixed_source_conversation_continuity": True, + } + assert functions["is_mixed_source_relevance_candidates_enabled"](without_chat) is False + assert functions["is_mixed_source_conversation_continuity_enabled"](without_chat) is False + with_chat = {**without_chat, "enable_mixed_source_chat_search": True} + assert functions["is_mixed_source_relevance_candidates_enabled"](with_chat) is True + assert functions["is_mixed_source_conversation_continuity_enabled"](with_chat) is True + print(" Subordinate switches follow their parent.") + return True + + +def test_kill_switch_applies_at_read_time_only(): + """The env kill switch overrides each read and never reaches the stored document.""" + print("\nTesting the SIMPLECHAT_DISABLE_MIXED_SOURCE kill switch...") + functions = load_settings_functions() + assert functions["MIXED_SOURCE_KILL_SWITCH_ENV"] == KILL_SWITCH + defaults = _defaults_for_simulation() + stored = {"enable_enhanced_citations": True, "mixed_source_settings_version": 1} + + original = os.environ.get(KILL_SWITCH) + try: + for value in ("1", "true", "YES", " on "): + os.environ[KILL_SWITCH] = value + persisted, returned = simulate_settings_load(functions, stored, defaults) + assert all(persisted[key] is True for key in DERIVED_KEYS), ( + "The kill switch must not change what is persisted" + ) + assert all(returned[key] is False for key in DERIVED_KEYS), ( + f"The kill switch value {value!r} must turn the derived behaviors off" + ) + for value in ("", "0", "false", "off"): + os.environ[KILL_SWITCH] = value + _, returned = simulate_settings_load(functions, stored, defaults) + assert all(returned[key] is True for key in DERIVED_KEYS) + finally: + if original is None: + os.environ.pop(KILL_SWITCH, None) + else: + os.environ[KILL_SWITCH] = original + + # Wiring: applied to every read, and only there. + format_result = _source_segment("_format_result", nested_in="get_settings") + loader = _source_segment("normalize_loaded_settings", nested_in="get_settings") + updater = _source_segment("update_settings") + assert "_apply_mixed_source_env_kill_switch(settings_payload)" in format_result + assert "_apply_mixed_source_env_kill_switch" not in loader + assert "_apply_mixed_source_env_kill_switch" not in updater + print(" The kill switch is read-time only.") + return True + + +def test_loads_and_saves_both_derive(): + """Loading and saving both rewrite the derived values, like the tabular plugin flag.""" + print("\nTesting load and save wiring...") + loader = _source_segment("normalize_loaded_settings", nested_in="get_settings") + updater = _source_segment("update_settings") + assert "normalize_mixed_source_derived_settings(merged)" in loader + assert "normalize_mixed_source_settings_upgrade(merged)" in loader + assert "normalize_mixed_source_derived_settings(settings_item)" in updater + # The one-time upgrade must run only on load: a save is an administrator's choice. + assert "normalize_mixed_source_settings_upgrade" not in updater + print(" Loads and saves both derive; only loads upgrade.") + return True + + +def test_retired_switches_are_purged_and_gone_from_defaults(): + """The dead switches are removed from stored settings and never re-seeded.""" + print("\nTesting retired switch removal...") + functions = load_settings_functions() + for key in RETIRED_KEYS: + assert key in functions["RETIRED_SETTING_KEYS"], f"{key} is not retired" + + stored = {key: True for key in RETIRED_KEYS} + stored["enable_enhanced_citations"] = True + persisted, returned = simulate_settings_load(functions, stored, _defaults_for_simulation()) + for key in RETIRED_KEYS: + assert key not in persisted + assert key not in returned + + defaults = all_default_keys() + for key in RETIRED_KEYS: + assert key not in defaults, f"{key} is still seeded by get_settings()" + print(" Retired switches are purged.") + return True + + +def test_one_time_upgrade_respects_later_choices(): + """Relevance candidates turn on once; a later administrator choice is kept.""" + print("\nTesting the one-time mixed-source settings upgrade...") + functions = load_settings_functions() + defaults = _defaults_for_simulation() + assert defaults["enable_mixed_source_relevance_candidates"] is True + assert defaults["enable_mixed_source_analyze_all"] is False + assert defaults["enable_mixed_source_development_telemetry"] is False + assert defaults["mixed_source_settings_version"] == 0 + + # A deployment from before the upgrade: the inert switch stored off, the switch that + # did nothing visible stored on, no marker. + stored = { + "enable_enhanced_citations": True, + "enable_mixed_source_relevance_candidates": False, + "enable_mixed_source_analyze_all": True, + "enable_mixed_source_development_telemetry": True, + } + persisted, _ = simulate_settings_load(functions, stored, defaults) + assert persisted["enable_mixed_source_relevance_candidates"] is True + assert persisted["enable_mixed_source_analyze_all"] is False + assert persisted["mixed_source_settings_version"] == functions["MIXED_SOURCE_SETTINGS_VERSION"] + # Telemetry is a real choice the upgrade leaves alone. + assert persisted["enable_mixed_source_development_telemetry"] is True + + # The administrator turns relevance candidates off afterwards; it stays off. + persisted["enable_mixed_source_relevance_candidates"] = False + persisted["enable_mixed_source_analyze_all"] = True + reloaded, _ = simulate_settings_load(functions, persisted, defaults) + assert reloaded["enable_mixed_source_relevance_candidates"] is False + assert reloaded["enable_mixed_source_analyze_all"] is True + assert functions["normalize_mixed_source_settings_upgrade"](reloaded) is False + + # A brand-new deployment starts from the defaults and lands in the same place. + fresh, _ = simulate_settings_load(functions, {}, defaults) + assert fresh["enable_mixed_source_relevance_candidates"] is True + assert fresh["enable_mixed_source_analyze_all"] is False + assert all(fresh[key] is False for key in DERIVED_KEYS), "Enhanced Citations defaults off" + + # A malformed marker is treated as not yet upgraded rather than trusted. + for marker in (True, "1", None, 0): + candidate = {"mixed_source_settings_version": marker} + assert functions["normalize_mixed_source_settings_upgrade"](candidate) is True + assert candidate["mixed_source_settings_version"] == 1 + print(" The upgrade runs once and later choices stick.") + return True + + +def test_admin_surfaces_describe_the_two_real_choices(): + """Both interfaces describe the two real switches where they belong.""" + print("\nTesting admin declarations...") + fields_module = import_app_module("admin_settings_fields") + declared = {} + for section_id, field in fields_module.iter_fields(): + if field.get("key"): + declared.setdefault(field["key"], (section_id, field)) + + defaults = read_default_settings({ + "enable_mixed_source_relevance_candidates", + "enable_mixed_source_development_telemetry", + }) + expectations = { + "enable_mixed_source_relevance_candidates": ("enhanced-citations-section", "citation"), + "enable_mixed_source_development_telemetry": ("application-insights-section", "logging"), + } + for key, (section_id, pane_id) in expectations.items(): + assert key in declared, f"{key} is not declared" + actual_section, field = declared[key] + assert actual_section == section_id, f"{key} is declared under {actual_section}" + assert field["type"] == "switch" + assert field["default"] == defaults[key], f"{key} schema default differs from get_settings()" + help_text = str(field.get("help") or "") + assert len(help_text) > 120, f"{key} needs a description of what it does and costs" + assert field["label"].lower() not in help_text.lower() + pane = (PANES_DIR / f"{pane_id}.html").read_text(encoding="utf-8") + assert f'name="{key}"' in pane, f"{key} has no classic control in {pane_id}.html" + assert f'aria-describedby="{key}_help"' in pane + assert f'id="{key}_help"' in pane + + relevance = declared["enable_mixed_source_relevance_candidates"][1] + assert relevance["depends_on"] == {"key": "enable_enhanced_citations", "equals": True} + + suppressed = fields_module.SUPPRESSED_CAPABILITY_KEYS + for key in DERIVED_KEYS: + assert key in suppressed, f"{key} would be drawn as a switch that reverts" + assert key not in declared + assert "enable_enhanced_citations" in suppressed["enable_mixed_source_chat_search"] + assert KILL_SWITCH in suppressed["enable_mixed_source_chat_search"] + assert "enable_mixed_source_analyze_all" in suppressed + assert "enable_mixed_source_analyze_all" not in declared + + citations_help = declared["enable_enhanced_citations"][1]["help"] + assert "spreadsheet" in citations_help.lower(), ( + "Enhanced Citations must say it also turns on spreadsheet analysis" + ) + print(" Both real choices are described where they belong.") + return True + + +def test_classic_save_persists_both_switches(): + """The server-rendered form writes both keys, so neither interface drifts.""" + print("\nTesting the classic save handler...") + source = ADMIN_ROUTE_FILE.read_text(encoding="utf-8") + for key in ( + "enable_mixed_source_relevance_candidates", + "enable_mixed_source_development_telemetry", + ): + assert f"'{key}': form_data.get('{key}') == 'on'" in source, f"{key} is not saved" + print(" The classic save persists both switches.") + return True + + +def test_compare_limitation_names_its_prerequisite(): + """Without spreadsheet processing, mixed Compare says what to enable.""" + print("\nTesting the cross-format Compare limitation message...") + source = WORKFLOW_RUNNER_FILE.read_text(encoding="utf-8") + assert "requires spreadsheet processing" in source + assert "Enhanced Citations" in source + assert "while cross-format Compare is disabled" not in source + print(" The limitation names Enhanced Citations.") + return True + + +if __name__ == "__main__": + tests = [ + test_derived_behaviors_follow_enhanced_citations, + test_subordinate_switches_need_their_parent, + test_kill_switch_applies_at_read_time_only, + test_loads_and_saves_both_derive, + test_retired_switches_are_purged_and_gone_from_defaults, + test_one_time_upgrade_respects_later_choices, + test_admin_surfaces_describe_the_two_real_choices, + test_classic_save_persists_both_switches, + test_compare_limitation_names_its_prerequisite, + ] + results = [] + for test in tests: + try: + results.append(bool(test())) + except Exception as error: + print(f"FAILED {test.__name__}: {error}") + import traceback + + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_orchestration_single_contract.py b/functional_tests/test_orchestration_single_contract.py index 7e0b4035a..4b6e970bb 100644 --- a/functional_tests/test_orchestration_single_contract.py +++ b/functional_tests/test_orchestration_single_contract.py @@ -1,7 +1,7 @@ # test_orchestration_single_contract.py """ Functional tests for Gather / Reason / Render as the only chat orchestration contract. -Version: 0.261.191 +Version: 0.261.260 Implemented in: 0.261.139 Per-source preparation guidance for mixed comparisons covered in: 0.261.191 @@ -108,7 +108,8 @@ def test_settings_defaults_no_longer_offer_the_toggle(modules): sanitize = source.split("def sanitize_settings_for_user(", 1)[1].split("\ndef ", 1)[0] assert TOGGLE not in defaults assert TOGGLE not in sanitize - assert settings.RETIRED_SETTING_KEYS == (TOGGLE,) + # The tuple also retires the mixed-source switches removed in 0.261.260. + assert TOGGLE in settings.RETIRED_SETTING_KEYS # Loading and saving both retire the stored switch. assert source.count("normalize_retired_orchestration_settings(") == 3 diff --git a/functional_tests/test_tabular_document_actions_workflow.py b/functional_tests/test_tabular_document_actions_workflow.py index 5e68b2afd..1d4754aae 100644 --- a/functional_tests/test_tabular_document_actions_workflow.py +++ b/functional_tests/test_tabular_document_actions_workflow.py @@ -2,8 +2,8 @@ # test_tabular_document_actions_workflow.py """ Functional test for tabular document-action workflow support. -Version: 0.250.185 -Implemented in: 0.241.038; mixed-source manifest coverage added in 0.250.062; generated-output Analyze durable routing added in 0.250.184; model endpoint context added in 0.250.185 +Version: 0.261.260 +Implemented in: 0.241.038; mixed-source manifest coverage added in 0.250.062; generated-output Analyze durable routing added in 0.250.184; model endpoint context added in 0.250.185; shadow manifest retired in 0.261.260 This test ensures tabular document actions reuse the shared tabular analysis path for Analyze and comparison workflows instead of relying only on the @@ -60,8 +60,8 @@ def test_shared_tabular_document_action_helper_exists() -> None: assert 'resolve_authorized_source_manifest(' in workflow_runner_content, ( "Expected workflow document actions to support the authorized Phase 1 source manifest." ) - assert 'is_mixed_source_manifest_enabled(settings)' in workflow_runner_content, ( - "Expected Phase 1 workflow manifest production to remain behind its internal flag." + assert 'is_mixed_source_manifest_enabled' not in workflow_runner_content, ( + "The Phase 1 shadow manifest was retired in 0.261.260 and must not return." ) assert 'augment_tabular_invocations_with_related_document_evidence(' in workflow_runner_content, ( "Expected the shared helper to reuse row-linked related-document augmentation for tabular workflows." @@ -242,7 +242,6 @@ def fake_maybe_queue_direct_tabular_generated_output(**kwargs): "TABULAR_PARITY_EVENT_FIRST_FOREGROUND_TABULAR_INVOCATION": "foreground_invocation", "raise_if_mixed_source_cancelled": orchestration.raise_if_mixed_source_cancelled, "is_tabular_processing_enabled": lambda settings: True, - "is_mixed_source_manifest_enabled": lambda settings: False, "_resolve_tabular_document_action_documents": lambda *args, **kwargs: [{ "document_id": "table-1", "document_name": "Bank Treasury Operations Dataset", @@ -337,8 +336,8 @@ def fail_invoke_prompt(*args, **kwargs): print("Generated-output tabular Analyze primary durable routing checks passed") -def test_manifest_flag_does_not_change_workflow_dispatch() -> None: - print("Testing workflow manifest flag behavior equivalence...") +def test_workflow_dispatch_no_longer_resolves_a_shadow_manifest() -> None: + print("Testing that workflow dispatch resolves no shadow manifest...") workflow_runner_tree = ast.parse(read_text(WORKFLOW_RUNNER_FILE)) helper_node = next( @@ -357,13 +356,6 @@ def test_manifest_flag_does_not_change_workflow_dispatch() -> None: "DOCUMENT_ACTION_TYPE_COMPARISON": "comparison", "raise_if_mixed_source_cancelled": orchestration.raise_if_mixed_source_cancelled, "is_tabular_processing_enabled": lambda settings: True, - "is_mixed_source_manifest_enabled": lambda settings: bool( - settings.get("enable_mixed_source_manifest") - ), - "_get_document_action_source_ids": lambda action_config: ( - list(action_config.get("document_ids") or []), - {}, - ), "resolve_authorized_source_manifest": lambda *args, **kwargs: ( manifest_calls.append((args, kwargs)) or [] ), @@ -379,18 +371,9 @@ def test_manifest_flag_does_not_change_workflow_dispatch() -> None: action_config = {"type": "analyze", "document_ids": ["table-1"]} workflow = {"user_id": "user-1"} - disabled_result = helper( - "analyze", - workflow, - action_config, - {"enable_mixed_source_manifest": False}, - conversation_id="conversation-1", - invoke_prompt=lambda *args, **kwargs: None, - ) - disabled_legacy_call = legacy_resolver_calls[-1] - assert manifest_calls == [] - - enabled_result = helper( + # A deployment that saved the retired switch as on must not resolve a manifest + # just to throw it away. + result = helper( "analyze", workflow, action_config, @@ -398,15 +381,11 @@ def test_manifest_flag_does_not_change_workflow_dispatch() -> None: conversation_id="conversation-1", invoke_prompt=lambda *args, **kwargs: None, ) - enabled_legacy_call = legacy_resolver_calls[-1] - - assert disabled_result == enabled_result is None - assert disabled_legacy_call == enabled_legacy_call - assert len(manifest_calls) == 1 - assert manifest_calls[0][0] == (["table-1"],) + assert result is None + assert manifest_calls == [] + assert len(legacy_resolver_calls) == 1 namespace["is_tabular_processing_enabled"] = lambda settings: False - manifest_calls.clear() legacy_resolver_calls.clear() disabled_tabular_result = helper( "analyze", @@ -417,34 +396,20 @@ def test_manifest_flag_does_not_change_workflow_dispatch() -> None: invoke_prompt=lambda *args, **kwargs: None, ) assert disabled_tabular_result is None - assert len(manifest_calls) == 1 + assert manifest_calls == [] assert legacy_resolver_calls == [] - print("Workflow manifest flag behavior equivalence checks passed") + print("Workflow dispatch shadow-manifest retirement checks passed") -def test_document_action_chat_does_not_duplicate_shadow_manifest() -> None: - print("Testing document-action Chat manifest ownership...") +def test_chat_no_longer_defines_a_shadow_manifest() -> None: + print("Testing that Chat no longer resolves a shadow manifest...") - route_tree = ast.parse( - read_text(ROOT / "application" / "single_app" / "route_backend_chats.py") - ) - document_action_function = next( - node - for node in ast.walk(route_tree) - if isinstance(node, ast.FunctionDef) - and node.name == "execute_document_action_chat_request" - ) - manifest_calls = [ - node - for node in ast.walk(document_action_function) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "_maybe_resolve_chat_source_manifest" - ] - assert manifest_calls == [] + route_source = read_text(ROOT / "application" / "single_app" / "route_backend_chats.py") + assert "_maybe_resolve_chat_source_manifest" not in route_source + assert "is_mixed_source_manifest_enabled" not in route_source - print("Document-action Chat manifest ownership checks passed") + print("Chat shadow-manifest retirement checks passed") def run_tests() -> bool: @@ -454,8 +419,8 @@ def run_tests() -> bool: test_tabular_document_actions_stream_live_activity, test_mixed_sources_preserve_valid_tabular_partition, test_generated_output_tabular_analyze_queues_direct_output_before_foreground, - test_manifest_flag_does_not_change_workflow_dispatch, - test_document_action_chat_does_not_duplicate_shadow_manifest, + test_workflow_dispatch_no_longer_resolves_a_shadow_manifest, + test_chat_no_longer_defines_a_shadow_manifest, ] results = [] setup_module() diff --git a/functional_tests/test_v2_admin_capability_placement.py b/functional_tests/test_v2_admin_capability_placement.py index 8c45da801..c42f5ff86 100644 --- a/functional_tests/test_v2_admin_capability_placement.py +++ b/functional_tests/test_v2_admin_capability_placement.py @@ -2,7 +2,7 @@ #!/usr/bin/env python3 """ Functional test pinning where the V2 admin surface files each capability toggle. -Version: 0.261.122 +Version: 0.261.260 Implemented in: 0.261.047 Settings that ``admin_settings_fields.py`` does not describe are still shown in the @@ -45,6 +45,13 @@ "storage" in ``data-management-storage-section`` and appeared under Backup & Recovery, while ``enable_key_vault_secret_expiration_reminders`` matched nothing at all and fell into "Other capabilities". + +The mixed-source settings were the next case. Every ``enable_mixed_source_*`` key +matched only "source" in ``source-review-section``, so five unrelated switches sat +under Knowledge > Web & Research > Deep Research labelled with their key names, and +the cross-format Compare pair fell into "Other capabilities". Two of them did nothing +on their own. The derived ones are now suppressed, the two real choices are declared +where they belong, and ``test_mixed_source_keys_are_never_guessed`` keeps it that way. """ import re @@ -111,6 +118,13 @@ "enable_chat_completion_audio_cues": ("desktop-notifications-section", "audio-video"), "enable_video_file_support": ("video-intelligence-section", "audio-video"), "enable_enhanced_extraction": ("document-intelligence-section", "extraction"), + # Matched only "source" and landed in Deep Research, a web research feature. It + # extends the spreadsheet analysis Enhanced Citations turns on, so it lives there. + "enable_mixed_source_relevance_candidates": ("enhanced-citations-section", "citation"), + # Same "source" match. It reports to Application Insights, so it sits beside the + # global logging switch, which the scan had filed under Debug Logging on "logging". + "enable_mixed_source_development_telemetry": ("application-insights-section", "logging"), + "enable_appinsights_global_logging": ("application-insights-section", "logging"), } # Keys the scan must skip entirely, because they are not settings an @@ -118,10 +132,20 @@ EXPECTED_SUPPRESSED_CAPABILITIES = ( "enable_tabular_processing_plugin", "enable_enhanced_citations_mount", + # Derived from enable_enhanced_citations on every read and save. "enable_mixed_source_chat_search", "enable_mixed_source_conversation_continuity", + "enable_cross_format_compare", + "enable_cross_format_compare_one_to_many", + # Gates an Analyze target no chat or workflow screen can request. + "enable_mixed_source_analyze_all", ) +# Every mixed-source key the settings document still carries. Each must be declared +# or suppressed: the scan filed all of them under Deep Research on the word "source", +# or under "Other capabilities", with nothing but the key name to explain them. +MIXED_SOURCE_KEY_PREFIXES = ("enable_mixed_source_", "enable_cross_format_compare") + # Relocations with no server-rendered counterpart to check against. Both are # documented in ``V2_ONLY_FIELDS``, which is what the section assertion below reads # instead of a pane. @@ -317,6 +341,38 @@ def test_non_editable_capabilities_are_suppressed_not_declared(): return True +def test_mixed_source_keys_are_never_guessed(): + """Every mixed-source key is declared with a description or deliberately hidden.""" + print("\nTesting that no mixed-source capability is left to the fallback scan...") + + declared = fields_module.get_declared_setting_keys() + suppressed = set(fields_module.get_suppressed_capability_keys()) + sections = build_sections() + + guessed = [] + for key in read_capability_keys(): + if not key.startswith(MIXED_SOURCE_KEY_PREFIXES): + continue + if key in declared or key in suppressed: + continue + placement = place_capability(key, sections) + where = ( + f"{placement['group_label']} > {placement['tab_label']} > {placement['section_id']}" + if placement + else "Other capabilities" + ) + guessed.append(f"{key} -> {where}") + + assert not guessed, ( + "These mixed-source settings would be drawn by the fallback scan with only " + "their key name as a label. Declare each one with help text in the section " + "it belongs to, or suppress it with a reason:\n " + "\n ".join(guessed) + ) + + print(" Every mixed-source key is declared or suppressed.") + return True + + def test_suppressed_capabilities_are_real_settings_keys(): """A suppression for a key that no longer exists hides nothing and misleads.""" print("\nTesting that suppressed keys still exist in the settings document...") @@ -435,6 +491,7 @@ def test_relocated_capabilities_exist_in_their_v1_panes(): test_ported_heuristic_still_matches_the_renderer, test_described_groups_receive_no_guessed_capabilities, test_non_editable_capabilities_are_suppressed_not_declared, + test_mixed_source_keys_are_never_guessed, test_suppressed_capabilities_are_real_settings_keys, test_relocated_capabilities_are_declared_where_they_belong, test_v2_only_relocations_are_documented, diff --git a/functional_tests/test_workflow_loop_limits.py b/functional_tests/test_workflow_loop_limits.py index 1130d5885..c64844b33 100644 --- a/functional_tests/test_workflow_loop_limits.py +++ b/functional_tests/test_workflow_loop_limits.py @@ -191,6 +191,7 @@ class ClosedError(Exception): "normalize_key_vault_reminder_settings", "normalize_model_endpoint_identity_header_settings", "normalize_retired_orchestration_settings", + "normalize_mixed_source_derived_settings", ): namespace[name] = lambda _settings: None writer = _production_function("functions_settings.py", "update_settings", namespace) diff --git a/functional_tests/test_workflow_orchestration_limits_settings.py b/functional_tests/test_workflow_orchestration_limits_settings.py index b756f71c8..6ac502f23 100644 --- a/functional_tests/test_workflow_orchestration_limits_settings.py +++ b/functional_tests/test_workflow_orchestration_limits_settings.py @@ -140,6 +140,7 @@ class ClosedError(Exception): "normalize_key_vault_reminder_settings", "normalize_model_endpoint_identity_header_settings", "normalize_retired_orchestration_settings", + "normalize_mixed_source_derived_settings", ): namespace[name] = lambda _settings: None return _production_function("functions_settings.py", "update_settings", namespace) diff --git a/ui_tests/test_v2_admin_mixed_source_settings.py b/ui_tests/test_v2_admin_mixed_source_settings.py new file mode 100644 index 000000000..cdb2f3276 --- /dev/null +++ b/ui_tests/test_v2_admin_mixed_source_settings.py @@ -0,0 +1,208 @@ +# test_v2_admin_mixed_source_settings.py +""" +Browser coverage for where V2 Admin Settings shows the mixed-source settings. +Version: 0.261.260 +Implemented in: 0.261.260 + +Exercise the built application with the real field schema and intercepted APIs. +Before 0.261.260 the page filed five mixed-source switches under Knowledge > Web & +Research > Deep Research, labelled only with their key names, because nothing declared +them and the fallback scan matched the word "source". Check that Deep Research and the +rest of the page now show none of them, that the two real choices appear with their +descriptions under Enhanced Citations and Application Insights, that the spreadsheet +switch is shown only while Enhanced Citations is on, that the behaviors derived from +Enhanced Citations are never drawn, and that both switches save through the production +field normalizer. The classic panes are rendered from their templates and must submit +both switches as normal form checkboxes, including the spreadsheet switch while Enhanced +Citations is off and its container is hidden. No live settings are written. +""" + +import re +import sys +from pathlib import Path + +import pytest +from jinja2 import Environment, FileSystemLoader, select_autoescape +from playwright.sync_api import expect + +sys.path.insert(0, str(Path(__file__).resolve().parent / "fixtures")) + +from v2_admin_settings import AdminSettingsFixture, connect_options # noqa: E402,F401 + + +pytestmark = pytest.mark.ui + +APP_ROOT = Path(__file__).resolve().parents[1] / "application" / "single_app" +MIXED_SOURCE_SECTIONS = { + "chat": ("enhanced-citations-section",), + "knowledge": ("source-review-section",), + "operations": ("application-insights-section",), +} +# A settings document after the 0.261.260 load: the derived behaviors follow Enhanced +# Citations and Analyze All has been reset. All of them are enable_* booleans the +# fallback scan would draw if they were not suppressed. +STORED_MIXED_SOURCE_SETTINGS = { + "enable_enhanced_citations": True, + "enable_mixed_source_chat_search": True, + "enable_mixed_source_conversation_continuity": True, + "enable_cross_format_compare": True, + "enable_cross_format_compare_one_to_many": True, + "enable_mixed_source_analyze_all": False, +} +CITATIONS_LABEL = "Enable Enhanced Citations" +RELEVANCE_KEY = "enable_mixed_source_relevance_candidates" +RELEVANCE_LABEL = "Look for relevant spreadsheets when no files are selected" +TELEMETRY_KEY = "enable_mixed_source_development_telemetry" +TELEMETRY_LABEL = "Record mixed document and spreadsheet metrics" +GLOBAL_LOGGING_LABEL = "Enable Application Insights Global Logging" +ARTIFACTS = "v2_admin_mixed_source" + + +@pytest.fixture +def mixed_source_ui(page): + fixture = AdminSettingsFixture(page, sections=MIXED_SOURCE_SECTIONS, validate_updates=True) + fixture.settings.update(STORED_MIXED_SOURCE_SETTINGS) + yield fixture + fixture.assert_clean() + + +def _switch(scope, label): + return scope.get_by_role("checkbox", name=re.compile(f"^{re.escape(label)}")) + + +def _set_switch(page, label, checked): + checkbox = _switch(page, label) + if checkbox.is_checked() != checked: + page.get_by_text(label, exact=True).click() + expect(checkbox).to_be_checked(checked=checked) + + +def _save(page): + page.get_by_role("button", name="Save changes", exact=True).click() + expect(page.get_by_role("button", name="Save changes", exact=True)).to_have_count(0) + + +def test_mixed_source_switches_are_described_where_they_belong(mixed_source_ui): + mixed_source_ui.open(ready_region="Enhanced") + page = mixed_source_ui.page + + deep_research = page.get_by_role("region", name="Deep Research", exact=True) + expect(deep_research).to_be_visible() + expect(deep_research.get_by_text(re.compile("mixed source", re.IGNORECASE))).to_have_count(0) + # A guessed row prints its raw key; a declared field prints its description instead. + expect(page.get_by_text(re.compile(r"enable_(mixed_source|cross_format)"))).to_have_count(0) + expect(page.get_by_text(re.compile(r"^(Mixed source|Cross format compare)", re.IGNORECASE))).to_have_count(0) + # The fixture always seeds the Agents switches, whose sections are not served here, + # so "Other capabilities" may exist; it must not hold a mixed-source row. + other = page.get_by_role("region", name="Other capabilities", exact=True) + expect(other.get_by_text(re.compile("mixed source|cross format", re.IGNORECASE))).to_have_count(0) + + enhanced = page.get_by_role("region", name="Enhanced", exact=True) + expect(enhanced.get_by_text("It also turns on spreadsheet analysis", exact=False)).to_be_visible() + relevance = _switch(enhanced, RELEVANCE_LABEL) + expect(relevance).to_be_visible() + expect(relevance).to_be_checked() + expect(enhanced.get_by_text("second search aimed at spreadsheet columns", exact=False)).to_be_visible() + + insights = page.get_by_role("region", name="Application Insights", exact=True) + expect(_switch(insights, GLOBAL_LOGGING_LABEL)).to_be_visible() + telemetry = _switch(insights, TELEMETRY_LABEL) + expect(telemetry).to_be_visible() + expect(telemetry).not_to_be_checked() + expect(insights.get_by_text("Never records prompts", exact=False)).to_be_visible() + mixed_source_ui.capture("placement", ARTIFACTS) + + +def test_spreadsheet_switch_is_shown_only_with_enhanced_citations(mixed_source_ui): + mixed_source_ui.open(ready_region="Enhanced") + page = mixed_source_ui.page + enhanced = page.get_by_role("region", name="Enhanced", exact=True) + + _set_switch(page, CITATIONS_LABEL, False) + expect(_switch(enhanced, RELEVANCE_LABEL)).to_have_count(0) + + _set_switch(page, CITATIONS_LABEL, True) + expect(_switch(enhanced, RELEVANCE_LABEL)).to_be_visible() + expect(_switch(enhanced, RELEVANCE_LABEL)).to_be_checked() + + +def test_spreadsheet_switch_saves_through_the_field_normalizer(mixed_source_ui): + mixed_source_ui.open(ready_region="Enhanced") + page = mixed_source_ui.page + + _set_switch(page, RELEVANCE_LABEL, False) + _save(page) + assert mixed_source_ui.patches == [{RELEVANCE_KEY: False}] + assert mixed_source_ui.settings[RELEVANCE_KEY] is False + + page.reload(wait_until="networkidle") + expect(_switch(page, RELEVANCE_LABEL)).not_to_be_checked() + + +def test_telemetry_switch_saves_through_the_field_normalizer(mixed_source_ui): + mixed_source_ui.open(ready_region="Enhanced") + page = mixed_source_ui.page + + _set_switch(page, TELEMETRY_LABEL, True) + _save(page) + assert mixed_source_ui.patches == [{TELEMETRY_KEY: True}] + assert mixed_source_ui.settings[TELEMETRY_KEY] is True + + +def _render_classic_pane(page, pane_id, settings): + environment = Environment( + loader=FileSystemLoader(APP_ROOT / "templates"), + autoescape=select_autoescape(["html"]), + ) + markup = environment.get_template(f"admin/_panes/{pane_id}.html").render( + settings=settings, + admin_landing_tab=pane_id, + ) + page.set_content(f'
{markup}
') + + +def _submitted(page, key): + return page.evaluate( + "(key) => new FormData(document.getElementById('admin-settings-form')).get(key)", + key, + ) + + +@pytest.mark.parametrize("width", [1280, 390]) +@pytest.mark.parametrize("enabled", [False, True]) +def test_classic_citation_pane_submits_the_spreadsheet_switch(page, width, enabled): + page.set_viewport_size({"width": width, "height": 900}) + _render_classic_pane(page, "citation", {"enable_enhanced_citations": True, RELEVANCE_KEY: enabled}) + # admin_settings.js reveals the Enhanced Citations settings while the feature is on. + page.evaluate("() => { document.getElementById('enhanced_citation_settings').style.display = 'block'; }") + + expect(page.get_by_role("heading", name="Spreadsheets in Chat")).to_be_visible() + checkbox = page.get_by_label(RELEVANCE_LABEL, exact=True) + expect(checkbox).to_be_checked(checked=enabled) + expect(checkbox).to_have_attribute("aria-describedby", f"{RELEVANCE_KEY}_help") + expect(page.locator(f"#{RELEVANCE_KEY}_help")).to_contain_text("second search aimed at spreadsheet columns") + + checkbox.check() + submitted_checked = _submitted(page, RELEVANCE_KEY) + checkbox.uncheck() + submitted_unchecked = _submitted(page, RELEVANCE_KEY) + assert submitted_checked == "on" + assert submitted_unchecked is None + + +def test_classic_save_keeps_the_spreadsheet_switch_while_citations_are_off(page): + """The switch sits in a hidden container then, and must still submit its stored value.""" + _render_classic_pane(page, "citation", {"enable_enhanced_citations": False, RELEVANCE_KEY: True}) + expect(page.get_by_label(RELEVANCE_LABEL, exact=True)).to_be_hidden() + assert _submitted(page, RELEVANCE_KEY) == "on" + + +def test_classic_logging_pane_submits_the_metrics_switch(page): + _render_classic_pane(page, "logging", {TELEMETRY_KEY: False}) + checkbox = page.get_by_label(TELEMETRY_LABEL, exact=True) + expect(checkbox).not_to_be_checked() + expect(checkbox).to_have_attribute("aria-describedby", f"{TELEMETRY_KEY}_help") + expect(page.locator(f"#{TELEMETRY_KEY}_help")).to_contain_text("Never records prompts") + + checkbox.check() + assert _submitted(page, TELEMETRY_KEY) == "on"