From bc55a075ebb4bbda9447dbc1522d4b43d0fa8579 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 19:22:10 -0400 Subject: [PATCH 01/13] Add V2 spreadsheet merge capability (Phase 1, refs #1619) Adds a deterministic tabular_merge Reason capability to V2 chat orchestration so several same-structure CSV or Excel files can be combined into one retained table and rendered to CSV or XLSX through render_file. - functions_tabular_merge.py: pure merge engine (by_name/exact_order policies, Source File provenance column, row limits, exact counts) - functions_orchestration_merge.py: limits, authorized byte loading, records-v1/report persistence and failure-code mapping - Registry, schema, adapter and executor wiring with catalog failure text - Exact tabular export profiles so CSV/XLSX render keeps every column - Admin Merge settings (enabled, chat/workflow document and row limits) - Docs, guides, release notes and functional/UI tests; goldens refreshed Version 0.261.218 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 77 +- application/single_app/config.py | 2 +- .../single_app/functions_document_actions.py | 72 +- .../functions_generated_export_registry.py | 17 +- .../functions_generated_office_adapters.py | 12 +- .../functions_orchestration_adapters.py | 96 ++ .../functions_orchestration_deliverables.py | 10 + .../functions_orchestration_executor.py | 9 +- .../functions_orchestration_merge.py | 168 +++ .../functions_orchestration_planner.py | 5 +- .../functions_orchestration_registry.py | 89 ++ ...unctions_orchestration_result_contracts.py | 2 +- .../functions_orchestration_schema.py | 44 + .../functions_structured_file_renderers.py | 16 +- .../single_app/functions_tabular_merge.py | 994 ++++++++++++++++++ .../route_frontend_admin_settings.py | 27 + .../templates/admin/_panes/actions.html | 86 +- docs/admin/agents-actions.md | 15 +- docs/admin/orchestration.md | 14 + .../features/CHAT_ORCHESTRATION.md | 1 + .../GENERATED_FILE_EXPORT_FRAMEWORK.md | 14 +- docs/explanation/features/V2_FILE_MERGE.md | 132 +++ .../V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md | 163 +++ docs/explanation/release_notes.md | 16 + .../guides/create-files-with-orchestration.md | 4 +- docs/guides/index.md | 1 + docs/guides/merge-files.md | 91 ++ ...test_admin_settings_pane_variable_scope.py | 33 +- ..._orchestration_tabular_merge_capability.py | 296 ++++++ ...estration_workflow_results_off_golden.json | 370 ++++++- ...rchestration_workflow_runs_off_golden.json | 182 +++- ...estration_workflow_setting_off_golden.json | 90 +- functional_tests/test_tabular_merge_engine.py | 438 ++++++++ .../test_v2_admin_actions_parity.py | 54 +- ...admin_document_action_capabilities_card.py | 23 +- 35 files changed, 3608 insertions(+), 55 deletions(-) create mode 100644 application/single_app/functions_orchestration_merge.py create mode 100644 application/single_app/functions_tabular_merge.py create mode 100644 docs/explanation/features/V2_FILE_MERGE.md create mode 100644 docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md create mode 100644 docs/guides/merge-files.md create mode 100644 functional_tests/test_orchestration_tabular_merge_capability.py create mode 100644 functional_tests/test_tabular_merge_engine.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 03bcec12d..6fc1246b1 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -310,6 +310,13 @@ DOCUMENT_ACTION_WORKFLOW_MIN_LIMIT = 2 DOCUMENT_ACTION_WORKFLOW_MAX_LIMIT = 1000 DOCUMENT_ACTION_WORKFLOW_DEFAULT_LIMIT = 10 +# Merge, mirrored from MERGE_* in functions_document_actions.py. +DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_LIMIT = 10 +DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_LIMIT = 100 +DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT = 1000 +DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT = 1_000_000 +DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_ROWS = 250_000 +DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_ROWS = 1_000_000 # Schemes permitted for administrator-configured navigation links. These render # into an anchor href, so allowing arbitrary schemes would let a saved link @@ -3820,6 +3827,7 @@ {"value": "document_analyze", "label": "Analyse documents"}, {"value": "document_compare", "label": "Compare documents"}, {"value": "tabular_analyze", "label": "Analyse spreadsheets"}, + {"value": "tabular_merge", "label": "Merge spreadsheets"}, {"value": "web_search", "label": "Search the web"}, {"value": "url_fetch", "label": "Read linked pages"}, {"value": "deep_research", "label": "Research in depth"}, @@ -4607,6 +4615,73 @@ "group": "Comparison", "depends_on": {"key": "document_action_comparison_enabled", "equals": True}, }, + { + "key": "document_action_merge_enabled", + "settings_path": ["document_action_capabilities", "merge", "enabled"], + "type": "switch", + "label": "Enable Merge", + "help": ( + "Lets chat orchestration combine the rows of several CSV or Excel files " + "that share the same columns into one table, then deliver it as a CSV or " + "Excel file. Rows are appended exactly by code, not rewritten by a model." + ), + "default": True, + "group": "Merge", + }, + { + "key": "document_action_merge_chat_max_documents", + "settings_path": ["document_action_capabilities", "merge", "chat_max_documents"], + "type": "number", + "label": "Merge: Chat Document Limit", + "help": ( + "Most files one chat request may merge. Merging is cheaper than Analyze " + "because no model reads the files, so this can be higher." + ), + "default": DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_LIMIT, + "min": DOCUMENT_ACTION_CHAT_MIN_LIMIT, + "max": DOCUMENT_ACTION_CHAT_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_workflow_max_documents", + "settings_path": ["document_action_capabilities", "merge", "workflow_max_documents"], + "type": "number", + "label": "Merge: Workflow Document Limit", + "help": "The same limit for a workflow run, where large batches belong.", + "default": DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_LIMIT, + "min": DOCUMENT_ACTION_WORKFLOW_MIN_LIMIT, + "max": DOCUMENT_ACTION_WORKFLOW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_chat_max_rows", + "settings_path": ["document_action_capabilities", "merge", "chat_max_rows"], + "type": "number", + "label": "Merge: Chat Row Limit", + "help": ( + "Most rows one chat merge may produce. A few very large files can cost more " + "than many small ones, so rows are limited as well as files." + ), + "default": DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_ROWS, + "min": DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT, + "max": DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_workflow_max_rows", + "settings_path": ["document_action_capabilities", "merge", "workflow_max_rows"], + "type": "number", + "label": "Merge: Workflow Row Limit", + "help": "The same row limit for a workflow run.", + "default": DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_ROWS, + "min": DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT, + "max": DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, ], # Rendered by V1 only while Workspace Mode is on, matching agent-toggles-card. "plugin-feature-toggles": [ @@ -7111,7 +7186,7 @@ def _apply_nested_paths(normalized, current_settings, warnings=None): Deep Research reading the stale copy. ``settings_path`` names a single destination as a list of segments. - ``document_action_capabilities`` holds six values across two action types, + ``document_action_capabilities`` holds the Analyze, Comparison and Merge values, and nothing reads a flattened form of them, so writing the flat keys through would save a setting the application never looks at. diff --git a/application/single_app/config.py b/application/single_app/config.py index 32111db14..4ef56809c 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -100,7 +100,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.217" +VERSION = "0.261.218" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_document_actions.py b/application/single_app/functions_document_actions.py index aa6652ff8..8b417f7eb 100644 --- a/application/single_app/functions_document_actions.py +++ b/application/single_app/functions_document_actions.py @@ -16,6 +16,7 @@ DOCUMENT_ACTION_TYPE_SEARCH = 'search' DOCUMENT_ACTION_TYPE_ANALYZE = 'analyze' DOCUMENT_ACTION_TYPE_COMPARISON = 'comparison' +DOCUMENT_ACTION_TYPE_MERGE = 'merge' DOCUMENT_ACTION_ANALYSIS_MODE_COMBINED = 'combined' DOCUMENT_ACTION_ANALYSIS_MODE_PER_DOCUMENT = 'per_document' DOCUMENT_ACTION_TARGET_MODE_ALL = 'all' @@ -38,6 +39,7 @@ DOCUMENT_ACTION_TYPE_SEARCH, DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON, + DOCUMENT_ACTION_TYPE_MERGE, } DOCUMENT_ACTION_LIMIT_BOUNDS = { DOCUMENT_ACTION_CONTEXT_CHAT: { @@ -49,6 +51,13 @@ 'max': 1000, }, } +# Merging is bounded by rows as well as files: a few very large files can cost more than +# many small ones. CSV exports allow at most 1,000,000 records. +MERGE_ROW_LIMIT_BOUNDS = {'min': 1000, 'max': 1_000_000} +MERGE_CHAT_DEFAULT_MAX_DOCUMENTS = 10 +MERGE_WORKFLOW_DEFAULT_MAX_DOCUMENTS = 100 +MERGE_CHAT_DEFAULT_MAX_ROWS = 250_000 +MERGE_WORKFLOW_DEFAULT_MAX_ROWS = 1_000_000 DEFAULT_DOCUMENT_ACTION_CAPABILITIES = { DOCUMENT_ACTION_TYPE_ANALYZE: { 'enabled': True, @@ -60,7 +69,21 @@ 'chat_max_documents': CHAT_DOCUMENT_ANALYSIS_MAX_DOCUMENTS, 'workflow_max_documents': WORKFLOW_DOCUMENT_ANALYSIS_MAX_DOCUMENTS, }, + DOCUMENT_ACTION_TYPE_MERGE: { + 'enabled': True, + 'chat_max_documents': MERGE_CHAT_DEFAULT_MAX_DOCUMENTS, + 'workflow_max_documents': MERGE_WORKFLOW_DEFAULT_MAX_DOCUMENTS, + 'chat_max_rows': MERGE_CHAT_DEFAULT_MAX_ROWS, + 'workflow_max_rows': MERGE_WORKFLOW_DEFAULT_MAX_ROWS, + }, } +# Classic chat and its document-action payloads only know Analyze and Comparison. Merge is +# reached through V2 orchestration and, later, workflow tasks, so it is never offered to a +# caller that does not ask for it explicitly. +DOCUMENT_ACTION_TYPES_WITH_PAYLOAD_SUPPORT = ( + DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_COMPARISON, +) def normalize_document_action_type(action_type): @@ -108,6 +131,14 @@ def _coerce_document_action_limit(value, default_value, execution_context): return max(bounds['min'], min(bounds['max'], normalized_value)) +def _coerce_merge_row_limit(value, default_value): + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = int(default_value) + return max(MERGE_ROW_LIMIT_BOUNDS['min'], min(MERGE_ROW_LIMIT_BOUNDS['max'], normalized_value)) + + def get_default_document_action_capabilities(): return copy.deepcopy(DEFAULT_DOCUMENT_ACTION_CAPABILITIES) @@ -138,6 +169,15 @@ def normalize_document_action_capabilities(settings_or_capabilities=None): DOCUMENT_ACTION_CONTEXT_WORKFLOW, ), } + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + normalized_capabilities[action_type].update({ + 'chat_max_rows': _coerce_merge_row_limit( + raw_capability.get('chat_max_rows'), default_capability['chat_max_rows'], + ), + 'workflow_max_rows': _coerce_merge_row_limit( + raw_capability.get('workflow_max_rows'), default_capability['workflow_max_rows'], + ), + }) return normalized_capabilities @@ -158,14 +198,32 @@ def is_document_action_enabled(action_type, settings=None): return bool(capability.get('enabled', False)) -def get_enabled_document_action_types(settings=None): +def get_enabled_document_action_types(settings=None, *, include_merge=False): + """Enabled action types for a payload owner. + + ``include_merge`` is opt-in so callers whose payloads cannot carry a merge action + (classic chat) never accept one. + """ enabled_action_types = {DOCUMENT_ACTION_TYPE_NONE, DOCUMENT_ACTION_TYPE_SEARCH} - for action_type in (DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON): + action_types = DOCUMENT_ACTION_TYPES_WITH_PAYLOAD_SUPPORT + ( + (DOCUMENT_ACTION_TYPE_MERGE,) if include_merge else () + ) + for action_type in action_types: if is_document_action_enabled(action_type, settings=settings): enabled_action_types.add(action_type) return enabled_action_types +def get_document_action_max_rows(action_type, execution_context, settings=None): + """The merged-row ceiling for Merge in chat or workflows; None for other actions.""" + if normalize_document_action_type(action_type) != DOCUMENT_ACTION_TYPE_MERGE: + return None + capability = get_document_action_capability(DOCUMENT_ACTION_TYPE_MERGE, settings=settings) + field_name = 'workflow_max_rows' if execution_context == DOCUMENT_ACTION_CONTEXT_WORKFLOW else 'chat_max_rows' + default_value = DEFAULT_DOCUMENT_ACTION_CAPABILITIES[DOCUMENT_ACTION_TYPE_MERGE][field_name] + return _coerce_merge_row_limit(capability.get(field_name, default_value), default_value) + + def get_document_action_max_documents(action_type, execution_context, settings=None): normalized_action_type = normalize_document_action_type(action_type) capability_action_type = DOCUMENT_ACTION_TYPE_ANALYZE if normalized_action_type == DOCUMENT_ACTION_TYPE_SEARCH else normalized_action_type @@ -193,6 +251,11 @@ def get_document_action_max_documents_by_type(execution_context, settings=None): execution_context, settings=settings, ), + DOCUMENT_ACTION_TYPE_MERGE: get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, + execution_context, + settings=settings, + ), } @@ -209,6 +272,8 @@ def _build_document_action_disabled_message(action_type): return 'Document analysis is currently disabled in admin settings.' if action_type == DOCUMENT_ACTION_TYPE_COMPARISON: return 'Document comparison is currently disabled in admin settings.' + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + return 'Merging files is not available here.' return 'The selected document action is currently disabled in admin settings.' @@ -304,6 +369,9 @@ def normalize_document_action_config( if action_type not in normalized_allowed_action_types: raise ValueError(_build_document_action_disabled_message(action_type)) + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + raise ValueError(_build_document_action_disabled_message(action_type)) + if action_type == DOCUMENT_ACTION_TYPE_ANALYZE: if 'analysis_options' in source_action or 'transformation_spec' in source_action: normalized_action['analysis_options'] = normalize_analysis_options( diff --git a/application/single_app/functions_generated_export_registry.py b/application/single_app/functions_generated_export_registry.py index 0942a99f2..357ad5a51 100644 --- a/application/single_app/functions_generated_export_registry.py +++ b/application/single_app/functions_generated_export_registry.py @@ -37,6 +37,10 @@ class GeneratedFileExportFormat: PREPARED_SLIDE_DECK_VERSION = 'prepared_slide_deck_v1' +EXACT_TABULAR_RECORDS_PROFILE = 'exact_tabular_records_v1' +EXACT_TABULAR_WORKBOOK_PROFILE = 'exact_tabular_workbook_v1' +EXACT_TABULAR_PROFILES = (EXACT_TABULAR_RECORDS_PROFILE, EXACT_TABULAR_WORKBOOK_PROFILE) +EXACT_TABULAR_DEFAULT_SHEET_NAME = 'Sheet1' GENERATED_IMAGE_REFERENCE_PATTERN = r'^asset:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' _OPTION_SCHEMAS = { 'columns': { @@ -161,7 +165,11 @@ class GeneratedFileExportFormat: GENERATED_FILE_EXPORT_REGISTRY = ( GeneratedFileExportFormat( 'csv', ('csv',), 'csv', 'text/csv; charset=utf-8', - (GeneratedFileExportProfile('tabular_records_v1', ('records',), ('columns',)),), + ( + GeneratedFileExportProfile('tabular_records_v1', ('records',), ('columns',)), + # Every retained column in retained order, for results whose schema is known only at run time. + GeneratedFileExportProfile(EXACT_TABULAR_RECORDS_PROFILE, ('records',)), + ), 'csv', ), GeneratedFileExportFormat( @@ -190,7 +198,10 @@ class GeneratedFileExportFormat: ), GeneratedFileExportFormat( 'xlsx', ('xlsx',), 'xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - (GeneratedFileExportProfile('tabular_workbook_v1', ('records',), ('columns', 'sheet_name')),), + ( + GeneratedFileExportProfile('tabular_workbook_v1', ('records',), ('columns', 'sheet_name')), + GeneratedFileExportProfile(EXACT_TABULAR_WORKBOOK_PROFILE, ('records',), optional_options=('sheet_name',)), + ), 'office', ('openpyxl',), ), GeneratedFileExportFormat( @@ -241,7 +252,7 @@ def _input_schema(profile): return {'type': 'string'} if profile.source_kinds == ('records',): record = {'type': 'object'} - if profile.profile in ('tabular_records_v1', 'tabular_workbook_v1'): + if profile.profile in ('tabular_records_v1', 'tabular_workbook_v1', *EXACT_TABULAR_PROFILES): record['additionalProperties'] = {'type': ['string', 'number', 'boolean', 'null']} return {'type': 'array', 'items': record} return {} diff --git a/application/single_app/functions_generated_office_adapters.py b/application/single_app/functions_generated_office_adapters.py index 768e42bc6..1ce347a16 100644 --- a/application/single_app/functions_generated_office_adapters.py +++ b/application/single_app/functions_generated_office_adapters.py @@ -18,6 +18,8 @@ GeneratedFileExportStream, ) from functions_generated_export_registry import ( + EXACT_TABULAR_DEFAULT_SHEET_NAME, + EXACT_TABULAR_WORKBOOK_PROFILE, GENERATED_IMAGE_REFERENCE_PATTERN, PREPARED_SLIDE_DECK_VERSION, get_prepared_slide_deck_schema, @@ -29,6 +31,7 @@ _SourceChecks, _iter_records, _render_text, + _tabular_request_columns, _validate_column_names, _validate_export_limits, _validate_source_limits, @@ -272,12 +275,17 @@ def _render_generated_office_source( rows = None with ExitStack() as resources: if entry.format_id == 'xlsx': - columns = _validate_column_names(request.columns, limits, checks, label='XLSX') + columns = _validate_column_names( + _tabular_request_columns(source, request), limits, checks, label='XLSX', + ) + sheet_name = request.sheet_name + if sheet_name is None and request.profile == EXACT_TABULAR_WORKBOOK_PROFILE: + sheet_name = EXACT_TABULAR_DEFAULT_SHEET_NAME rows = resources.enter_context(ClosingExportResource( boundary.rows(_workbook_records(source, request, columns, limits, checks)), )) prepared = office.PreparedWorkbook(( - office.PreparedSheet(request.sheet_name, columns, rows, checks.expected_count), + office.PreparedSheet(sheet_name, columns, rows, checks.expected_count), )) renderer = office.render_prepared_xlsx elif entry.format_id in ('docx', 'pdf'): diff --git a/application/single_app/functions_orchestration_adapters.py b/application/single_app/functions_orchestration_adapters.py index 934208d06..e1274556f 100644 --- a/application/single_app/functions_orchestration_adapters.py +++ b/application/single_app/functions_orchestration_adapters.py @@ -88,6 +88,7 @@ CAPABILITY_DOCUMENT_COMPARE, CAPABILITY_DOCUMENT_SEARCH, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_MERGE, CAPABILITY_URL_FETCH, CAPABILITY_WEB_SEARCH, CAPABILITY_WORKFLOW_PROPOSE, @@ -105,6 +106,7 @@ STEP_STATUS_COMPLETED, STEP_STATUS_FAILED, STEP_STATUS_PARTIAL, + access_failure, build_step_result, build_failure, failure_from_exception, @@ -931,6 +933,99 @@ def run_document_compare(step, context, *, settings, user_id, emit, cancel_reque ) +# -------------------------------------------------------------------------------------- +# tabular_merge -> functions_tabular_merge through functions_orchestration_merge +# -------------------------------------------------------------------------------------- + +def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requested): + """Append the rows of authorized same-structure spreadsheets; no model and no file. + + The retained ``records`` output is what ``render_file`` turns into a CSV or XLSX file. + """ + arguments = _arguments(step) + # The merge engine is standard-library only; its Excel readers load only when used. + from functions_tabular_merge import TabularMergeCancelled, TabularMergeError + + try: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge') + input_fingerprint = _internal_result_input_fingerprint(step, context) + from functions_orchestration_merge import ( + build_tabular_merge_sources, + merge_step_summary, + persist_tabular_merge_result, + require_tabular_merge_manifest, + tabular_merge_limits, + tabular_merge_options, + ) + from functions_tabular_merge import merge_tabular_sources + + with orchestration_file_policy(allow_generated_files=False): + document_ids = _string_list(arguments.get('document_ids')) + limits = tabular_merge_limits(settings) + if len(document_ids) > limits.max_sources: + # A bound source set is checked here; explicit IDs were checked at planning. + raise TabularMergeError('too_many_sources', 'Too many files were selected to merge here.') + options = tabular_merge_options(arguments) + service, producer, token, input_fingerprint = _internal_result_context( + step, context, user_id, CAPABILITY_TABULAR_MERGE, input_fingerprint=input_fingerprint, + ) + manifest = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, + ) + require_tabular_merge_manifest(manifest, document_ids) + _emit(emit, _progress(step, CAPABILITY_TABULAR_MERGE, 'Merging spreadsheets')) + sources = build_tabular_merge_sources( + manifest, user_id, byte_reader=_ctx(context, 'merge_source_reader', None), + ) + with merge_tabular_sources( + sources, options=options, limits=limits, + cancel_requested=lambda: _is_cancelled(cancel_requested), + ) as merge_result: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_saving') + task_result = persist_tabular_merge_result( + service=service, producer=producer, merge_result=merge_result, sources=manifest, + guard_token=token, input_fingerprint=input_fingerprint, + ) + summary = merge_step_summary(merge_result) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_finalization') + except (MixedSourceCancellationError, TabularMergeCancelled): + return _cancelled_result('Merging spreadsheets was cancelled.') + except TabularMergeError as exc: + from functions_orchestration_merge import merge_failure_code + + failure = build_failure(merge_failure_code(exc)) + log_event( + f'{_LOG_PREFIX} Spreadsheet merge was refused.', + extra={ + 'user_id': user_id, 'step_id': (step or {}).get('step_id'), + 'merge_code': exc.code, 'failure_code': failure['code'], + }, + level=logging.WARNING, + ) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + except Exception as exc: + # Typed authority, cancellation and lifecycle controls belong to the owning runtime. + from functions_orchestration_result_runtime import raise_source_service_failure + + raise_source_service_failure(exc) + log_event( + f'{_LOG_PREFIX} Spreadsheet merge failed.', + extra={'user_id': user_id, 'step_id': (step or {}).get('step_id'), 'error_type': type(exc).__name__}, + level=logging.ERROR, exceptionTraceback=True, + ) + if isinstance(exc, (PermissionError, LookupError)) or type(exc).__name__ in ( + 'ResultUnavailableError', 'DocumentHeldError', + ): + failure = access_failure(exc) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + return _failed_result('The spreadsheet merge could not be completed.', exc) + return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) + + # -------------------------------------------------------------------------------------- # tabular_analyze -> the native tabular bridge the server binds for each step # -------------------------------------------------------------------------------------- @@ -1890,6 +1985,7 @@ def run_workflow_results(step, context, *, settings, user_id, emit=None, cancel_ CAPABILITY_DOCUMENT_ANALYZE: run_document_analyze, CAPABILITY_DOCUMENT_COMPARE: run_document_compare, CAPABILITY_TABULAR_ANALYZE: run_tabular_analyze, + CAPABILITY_TABULAR_MERGE: run_tabular_merge, CAPABILITY_WEB_SEARCH: run_web_search, CAPABILITY_URL_FETCH: run_url_fetch, CAPABILITY_DEEP_RESEARCH: run_deep_research, diff --git a/application/single_app/functions_orchestration_deliverables.py b/application/single_app/functions_orchestration_deliverables.py index dc13c7d31..0f7a1062e 100644 --- a/application/single_app/functions_orchestration_deliverables.py +++ b/application/single_app/functions_orchestration_deliverables.py @@ -41,6 +41,7 @@ CAPABILITY_DOCUMENT_COMPARE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_RENDER_FILE, + CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, @@ -331,6 +332,15 @@ def build_deliverable_availability(settings, *, capabilities, unavailable=None, 'with pptx and prepared_slide_deck_v1.' )}, ]) + if CAPABILITY_TABULAR_MERGE in available: + recipes.append({'for': 'One file merged from several CSV or Excel files', 'steps': ( + 'tabular_merge with the files in merge order, then render_file binding its "records" ' + 'output with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1 ' + '(options.sheet_name is optional). Never compose merged rows.' + )}) + facts.append( + 'tabular_merge appends rows exactly; it does not match rows on a key column or change values.' + ) if explicit_images['status'] == 'available': facts.extend([ 'generate_image follows the visual style the user asks for, including photorealistic ' diff --git a/application/single_app/functions_orchestration_executor.py b/application/single_app/functions_orchestration_executor.py index 860d9e6c3..b3d4d8440 100644 --- a/application/single_app/functions_orchestration_executor.py +++ b/application/single_app/functions_orchestration_executor.py @@ -63,9 +63,9 @@ ) from functions_orchestration_deliverables import explicit_image_shortfalls from functions_orchestration_registry import ( - CAPABILITY_TABULAR_ANALYZE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, - DEPENDENCY_PLAN_CONTRACT_VERSION, admitted_export_pairs, external_effect_capability_ids, get_capability, - resolve_available_capability_ids, + CAPABILITY_TABULAR_ANALYZE, CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, + CAPABILITY_WORKFLOW_RUN, DEPENDENCY_PLAN_CONTRACT_VERSION, admitted_export_pairs, external_effect_capability_ids, + get_capability, resolve_available_capability_ids, ) from functions_orchestration_result_contracts import ( InputBinding, PartialInputNotAcceptedError, ProducerIdentity, ResultContractError, ResultRef, TaskResult, digest, @@ -866,7 +866,8 @@ def _run_dependency_step( readers = resolve_step_inputs(step, context) runtime_step = deepcopy(step) named_sources = [] - if 'sources' in readers and step['capability_id'] == 'document_analyze': + # Analyze and Merge accept a search's source set in place of explicit document IDs. + if 'sources' in readers and step['capability_id'] in ('document_analyze', CAPABILITY_TABULAR_MERGE): named_sources = read_complete_input(readers['sources']) runtime_step['arguments']['document_ids'] = [ source['document_id'] for source in named_sources diff --git a/application/single_app/functions_orchestration_merge.py b/application/single_app/functions_orchestration_merge.py new file mode 100644 index 000000000..ef70c3a01 --- /dev/null +++ b/application/single_app/functions_orchestration_merge.py @@ -0,0 +1,168 @@ +# functions_orchestration_merge.py +"""Deterministic spreadsheet merging for Gather / Reason / Render orchestration. + +Version: 0.261.218 + +``tabular_merge`` is a Reason capability. Its adapter, ``run_tabular_merge`` in +``functions_orchestration_adapters``, authorizes the sources. This module reads each +original file through the screening-aware byte reader, merges the files with +``functions_tabular_merge`` and retains the merged rows plus a merge report. It creates no +file and calls no model: ``render_file`` delivers a file from the retained ``records``. +""" + +from copy import deepcopy + +from content_screening.access import read_available_document_bytes +from functions_analysis_access import analysis_source_snapshot +from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_CHAT, + DOCUMENT_ACTION_TYPE_MERGE, + get_document_action_max_documents, + get_document_action_max_rows, +) +from functions_mixed_source_orchestration import AUTHORIZATION_STATUS_AUTHORIZED, SOURCE_KIND_TABULAR +from functions_orchestration_result_contracts import Completeness, Coverage, RecordColumn, ResultContractError +from functions_orchestration_results import NamedOutput +from functions_tabular_merge import ( + DEFAULT_SOURCE_COLUMN_NAME, + SCHEMA_POLICY_BY_NAME, + TabularMergeError, + TabularMergeLimits, + TabularMergeOptions, + TabularMergeSource, + tabular_merge_format_for_file_name, +) + + +TABULAR_MERGE_CAPABILITY_ID = "tabular_merge" +TABULAR_MERGE_SOURCE_PURPOSE = "native" +TABULAR_MERGE_CHECKS = ( + "authorized_source_snapshots", + "deterministic_row_append", + "exact_row_count", + "schema_policy_enforced", +) +# Engine codes become application-owned failure codes; their text never reaches a user. +MERGE_FAILURE_CODES = { + "schema_mismatch": "merge_schema_mismatch", + "duplicate_columns": "merge_columns_invalid", + "header_too_long": "merge_columns_invalid", + "too_many_columns": "merge_columns_invalid", + "empty_source": "merge_columns_invalid", + "row_has_extra_values": "merge_rows_invalid", + "cell_too_large": "merge_rows_invalid", + "unreadable_csv": "merge_source_unreadable", + "unreadable_workbook": "merge_source_unreadable", + "encrypted_workbook": "merge_source_unreadable", + "source_unavailable": "merge_source_unreadable", + "sheet_not_found": "merge_sheet_not_found", + "too_few_sources": "merge_sources_invalid", + "duplicate_source": "merge_sources_invalid", + "unsupported_format": "merge_sources_invalid", + "invalid_source": "merge_sources_invalid", + "too_many_sources": "merge_limit_exceeded", + "row_limit_exceeded": "merge_limit_exceeded", + "source_too_large": "merge_limit_exceeded", + "workbook_too_large": "merge_limit_exceeded", +} + + +def merge_failure_code(error): + """The application-owned failure code for an engine error, or the generic step failure.""" + return MERGE_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") + + +def tabular_merge_limits(settings, *, execution_context=DOCUMENT_ACTION_CONTEXT_CHAT): + """The administrator's Merge limits for chat or workflows as engine limits.""" + max_sources = get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings, + ) + max_rows = get_document_action_max_rows(DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings) + return TabularMergeLimits(max_sources=max(2, int(max_sources)), max_total_rows=int(max_rows)) + + +def tabular_merge_options(arguments): + """Engine options from validated step arguments.""" + arguments = arguments if isinstance(arguments, dict) else {} + return TabularMergeOptions( + schema_policy=arguments.get("schema_policy") or SCHEMA_POLICY_BY_NAME, + sheet=arguments.get("sheet") or None, + include_source_column=arguments.get("include_source_column", True) is not False, + source_column_name=arguments.get("source_column_name") or DEFAULT_SOURCE_COLUMN_NAME, + ) + + +def require_tabular_merge_manifest(manifest, document_ids): + """The authorized manifest in merge order, with every source a CSV or Excel file.""" + document_ids = list(document_ids or ()) + if ( + not isinstance(manifest, list) or len(document_ids) < 2 + or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids + or any(source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED for source in manifest) + ): + raise PermissionError("The complete merge source selection is unavailable.") + for source in manifest: + if ( + source.get("source_kind") != SOURCE_KIND_TABULAR + or tabular_merge_format_for_file_name(source.get("file_name")) is None + ): + raise TabularMergeError("unsupported_format", "Only CSV and Excel files can be merged.") + return manifest + + +def build_tabular_merge_sources(manifest, user_id, *, byte_reader=None): + """Engine sources whose bytes are read, one file at a time, through the access boundary.""" + reader = byte_reader or read_available_document_bytes + sources = [] + for entry in manifest: + source = deepcopy(entry) + + def load(source=source): + _document, content = reader( + source, user_id, source.get("group_id"), source.get("public_workspace_id"), + purpose=TABULAR_MERGE_SOURCE_PURPOSE, + ) + return content + + sources.append(TabularMergeSource( + source_id=source["document_id"], + file_name=source.get("file_name") or source.get("display_name") or source["document_id"], + load_bytes=load, + )) + return sources + + +def persist_tabular_merge_result(*, service, producer, merge_result, sources, guard_token, input_fingerprint): + """Retain the merged rows and the merge report as one complete Reason result.""" + if producer.capability_id != TABULAR_MERGE_CAPABILITY_ID: + raise ResultContractError("result_merge_invalid") + snapshots = analysis_source_snapshot(sources) + columns = tuple(RecordColumn(name, "string") for name in merge_result.columns) + row_count = merge_result.row_count + coverage = Coverage(len(snapshots), len(snapshots), "sources") + report = merge_result.report + limitations = tuple(report.get("limitations") or ()) + outputs = [ + NamedOutput( + "records", "records-v1", merge_result.iter_records(), + Completeness("complete", row_count, row_count, coverage, "valid", TABULAR_MERGE_CHECKS, limitations), + columns, + ), + NamedOutput( + "report", "structured-v1", report, + Completeness("complete", 1, 1, coverage, "valid", TABULAR_MERGE_CHECKS, limitations), + ), + ] + return service.persist_task_result( + producer=producer, role="reason", status="complete", outputs=outputs, + sources=snapshots, origin="grounded", guard_token=guard_token, input_fingerprint=input_fingerprint, + ) + + +def merge_step_summary(merge_result): + """A short, application-owned description of a completed merge.""" + totals = (merge_result.report or {}).get("totals") or {} + return ( + f"Merged {totals.get('rows', merge_result.row_count):,} row(s) from " + f"{totals.get('sources', 0)} file(s) into {totals.get('columns', len(merge_result.columns))} column(s)." + ) diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index ff456399f..1bdd8d060 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -421,7 +421,10 @@ def resolve_planner_client(settings): invent one. If the user already selected documents, plan around those documents. Use each candidate's server-resolved source_kind, not its display label, to choose compatible work. Never send tabular source IDs to document_analyze or document_compare: those steps -do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each +do not admit native tabular inputs. To combine the rows of several tabular sources that share +the same columns into one table or file, use tabular_merge with every one of those sources, never +compose; it appends rows and cannot match rows on a key, so ask when the user may mean that. +For mixed narrative/tabular comparisons, prepare each source with compatible offered capabilities and compose their named results. Ask each preparation step only for its own sources' contribution, such as values, periods, units and identifiers. A preparation step sees only its own sources, so never ask it to compare them diff --git a/application/single_app/functions_orchestration_registry.py b/application/single_app/functions_orchestration_registry.py index b31516ca8..917e76075 100644 --- a/application/single_app/functions_orchestration_registry.py +++ b/application/single_app/functions_orchestration_registry.py @@ -77,6 +77,7 @@ def effective_capability_ids(values): # produced the `_load_orchestration_helper` shim in `functions_tabular_analysis.py`. DOCUMENT_ACTION_TYPE_ANALYZE = 'analyze' DOCUMENT_ACTION_TYPE_COMPARISON = 'comparison' +DOCUMENT_ACTION_TYPE_MERGE = 'merge' DOCUMENT_ACTION_CONTEXT_CHAT = 'chat' COST_CLASS_LOW = 'low' @@ -96,6 +97,7 @@ def effective_capability_ids(values): CAPABILITY_DOCUMENT_ANALYZE = 'document_analyze' CAPABILITY_DOCUMENT_COMPARE = 'document_compare' CAPABILITY_TABULAR_ANALYZE = 'tabular_analyze' +CAPABILITY_TABULAR_MERGE = 'tabular_merge' CAPABILITY_WEB_SEARCH = 'web_search' CAPABILITY_URL_FETCH = 'url_fetch' CAPABILITY_DEEP_RESEARCH = 'deep_research' @@ -779,6 +781,93 @@ def generate_image_arguments_schema(options=None): 'max_per_plan': 2, 'adapter': CAPABILITY_TABULAR_ANALYZE, }, + { + 'id': CAPABILITY_TABULAR_MERGE, + 'label': 'Merge spreadsheets', + 'role': ROLE_REASON, + 'request_gate': None, + 'summary': ( + 'Combine the rows of two or more CSV or Excel files that share the same columns into ' + 'one table, exactly and without a model.' + ), + 'when_to_use': ( + 'Use when the user asks to merge, combine, append, stack or consolidate several CSV or ' + 'Excel files with the same columns into one table or file. Every row is kept exactly, in ' + 'the order the files are listed. It never matches rows on a key column (no joins or ' + 'lookups) and never edits values; when "merge" could mean matching rows on a key, ask. ' + 'Name the files in document_ids in merge order, or bind inputs.sources to a search ' + "step's sources output when the files must be found first. Columns are matched by name " + 'in any order; schema_policy exact_order also requires the same order. sheet picks one ' + "sheet in every workbook; otherwise each workbook's first visible sheet is read. " + 'include_source_column adds each row\'s file name. To deliver a file, bind the records ' + 'output to render_file with csv and exact_tabular_records_v1, or xlsx and ' + 'exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the ' + 'chat limit, or one that should repeat, belongs in a workflow.' + ), + 'settings_gates': (), + 'settings_gates_any': ( + 'enable_user_workspace', + 'enable_group_workspaces', + 'enable_public_workspaces', + ), + 'gate': _document_action_gate(DOCUMENT_ACTION_TYPE_MERGE), + 'requires_scope': (SCOPE_PERSONAL, SCOPE_GROUP, SCOPE_PUBLIC), + 'inputs': { + 'type': 'object', + 'properties': { + 'document_ids': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 2, + 'uniqueItems': True, + 'description': 'The CSV or Excel files to merge, in the order their rows should appear.', + }, + 'doc_scope': { + 'type': 'string', + 'enum': ['all', 'personal', 'group', 'public'], + 'default': 'all', + }, + 'schema_policy': { + 'type': 'string', + 'enum': ['by_name', 'exact_order'], + 'default': 'by_name', + 'description': ( + 'by_name accepts the same columns in any order; exact_order also requires ' + 'the same order.' + ), + }, + 'sheet': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 31, + 'description': "The sheet to read in every Excel file. Omit for each workbook's first visible sheet.", + }, + 'include_source_column': { + 'type': 'boolean', + 'default': True, + 'description': "Add a column holding each row's file name.", + }, + 'source_column_name': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 128, + 'default': 'Source File', + }, + }, + 'additionalProperties': False, + }, + 'result_contract_version': 'tabular-merge-v1', + 'result_outputs': {'records': 'records-v1', 'report': 'structured-v1'}, + # Files a search found can be merged by binding its source set by name. + 'result_input_kinds': {'sources': ('source-set-v1',)}, + 'partial_inputs_supported': False, + 'produces': (PRODUCES_RETAINED_RESULTS,), + 'cost_class': COST_CLASS_MEDIUM, + 'max_per_plan': 2, + 'adapter': CAPABILITY_TABULAR_MERGE, + # Enforced by the validator against the administrator's chat limit for Merge. + 'document_action_type': DOCUMENT_ACTION_TYPE_MERGE, + }, { 'id': CAPABILITY_WEB_SEARCH, 'label': 'Search the web', diff --git a/application/single_app/functions_orchestration_result_contracts.py b/application/single_app/functions_orchestration_result_contracts.py index cd1674a62..fd3d2708f 100644 --- a/application/single_app/functions_orchestration_result_contracts.py +++ b/application/single_app/functions_orchestration_result_contracts.py @@ -33,7 +33,7 @@ }) RESULT_STATES = frozenset({"pending", "partial", "complete", "invalid", "cancelled", "failed", "unavailable"}) RESULT_ROLES = frozenset({"gather", "reason", "render"}) -REASON_CAPABILITIES = frozenset({"document_analyze", "document_compare", "tabular_analyze"}) +REASON_CAPABILITIES = frozenset({"document_analyze", "document_compare", "tabular_analyze", "tabular_merge"}) MAX_OUTPUTS = 32 MAX_COLUMNS = 256 MAX_BINDING_STEPS = 64 diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index 60cae90c2..576323f79 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -56,6 +56,7 @@ CAPABILITY_COMPOSE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, @@ -871,6 +872,15 @@ def validate_dependency_plan( ) if capability_id == 'document_analyze' and arguments.get('document_ids') and 'sources' in step['inputs']: raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + if capability_id == CAPABILITY_TABULAR_MERGE: + if not arguments.get('document_ids') and 'sources' not in step['inputs']: + raise PlanValidationError( + 'Merge requires arguments.document_ids naming at least two CSV or Excel files in ' + 'merge order, or an inputs.sources source-set binding.', + code='source_binding_required', rule='merge_sources_required', + ) + if arguments.get('document_ids') and 'sources' in step['inputs']: + raise PlanValidationError('Use either explicit document IDs or a named source-set input.') accepted.append(step) if not any(step['enabled'] for step in accepted): raise PlanValidationError('The plan contains no enabled work.') @@ -1007,6 +1017,18 @@ def validate_plan_document_source_kinds(plan, document_source_kinds): if not isinstance(document_source_kinds, dict): raise PlanValidationError('Document type metadata is unavailable.', code='source_metadata_invalid') for step in plan.get('steps') or []: + if step.get('enabled', True) and step.get('capability_id') == CAPABILITY_TABULAR_MERGE: + if any( + document_source_kinds.get(document_id) not in (None, 'tabular') + for document_id in plan_document_ids({'steps': [step]}) + ): + raise PlanValidationError( + f'Step "{step["step_id"]}" merges only CSV or Excel files, and a selected source is ' + 'another kind of document. Merge only the tabular sources, or prepare the others ' + 'with compatible offered capabilities. Keep every selected source.', + code='source_kind_invalid', rule='tabular_source_required', + ) + continue if not step.get('enabled', True) or step.get('capability_id') not in ( 'document_analyze', 'document_compare', ): @@ -1509,6 +1531,28 @@ def summarize_plan(plan): 'image_generation_unavailable': 'Image generation is not available for this deployment right now.', 'image_request_invalid': 'The image model did not accept the planned image request.', 'retry_would_repeat': 'A retry would send the same request that was declined. Ask again with a different description instead.', + 'merge_schema_mismatch': ( + "The selected files don't all have the same columns, so nothing was merged. Ask to " + 'inspect their columns to see how they differ.' + ), + 'merge_columns_invalid': ( + 'A selected file has duplicate or overly long column headers, no header row, or too many ' + 'columns, so nothing was merged.' + ), + 'merge_rows_invalid': ( + 'A selected file has a row with more values than column headers, or a value longer than a ' + 'spreadsheet cell allows, so nothing was merged.' + ), + 'merge_source_unreadable': ( + "A selected file couldn't be read as a CSV or Excel file. It may be damaged or " + 'password-protected.' + ), + 'merge_sheet_not_found': "A selected workbook doesn't have the requested sheet, so nothing was merged.", + 'merge_sources_invalid': 'Merging needs at least two different CSV or Excel files (.csv, .xlsx, .xlsm or .xls).', + 'merge_limit_exceeded': ( + 'The merge is larger than chat allows. Merge fewer or smaller files, or ask for a workflow ' + 'to merge them.' + ), 'step_failed': 'This operation could not complete.', 'message_not_saved': 'The explanation could not be saved. Reload this run to check its durable status.', LEGACY_PLAN_CODE: LEGACY_PLAN_MESSAGE, diff --git a/application/single_app/functions_structured_file_renderers.py b/application/single_app/functions_structured_file_renderers.py index dda22c389..34ee0a7d2 100644 --- a/application/single_app/functions_structured_file_renderers.py +++ b/application/single_app/functions_structured_file_renderers.py @@ -21,7 +21,7 @@ GeneratedFileExportReadiness, GeneratedFileExportStream, ) -from functions_generated_export_registry import resolve_generated_file_export_format +from functions_generated_export_registry import EXACT_TABULAR_PROFILES, resolve_generated_file_export_format _CHUNK_CHARACTERS = 16384 @@ -252,6 +252,18 @@ def _validate_columns(columns, limits, checks): return tuple(columns), safe_columns +def _tabular_request_columns(source, request): + """Explicit projections name their columns; exact profiles write the retained schema in order.""" + if request.profile not in EXACT_TABULAR_PROFILES: + return request.columns + if request.columns is not None: + raise GeneratedFileExportError('invalid_options', 'Exact tabular exports write every retained column.') + columns = getattr(source, 'columns', None) + if type(columns) not in (tuple, list) or not columns: + raise GeneratedFileExportError('unsupported_source', 'Exact tabular exports need a retained column schema.') + return tuple(columns) + + def _csv_cell(value): if value is None: return '' @@ -265,7 +277,7 @@ def _csv_cell(value): def _render_csv(writer, source, request, limits, checks): - columns, safe_columns = _validate_columns(request.columns, limits, checks) + columns, safe_columns = _validate_columns(_tabular_request_columns(source, request), limits, checks) expected_keys = set(columns) csv_writer = csv.writer(writer, lineterminator='\r\n') csv_writer.writerow(safe_columns) diff --git a/application/single_app/functions_tabular_merge.py b/application/single_app/functions_tabular_merge.py new file mode 100644 index 000000000..ecb18e4e3 --- /dev/null +++ b/application/single_app/functions_tabular_merge.py @@ -0,0 +1,994 @@ +# functions_tabular_merge.py +"""Deterministic merging of same-structure CSV and Excel files into one table. + +Version: 0.261.218 + +The engine is pure: it receives already-authorized byte loaders, never resolves +documents, settings, storage, routes, or models, and performs no model work. +Values are kept as text so leading zeros, codes, and identifiers survive exactly. +Rows are spooled to a bounded temporary file during the single parse, so callers +learn the exact row count before they persist or render the merged result. +""" + +import codecs +import csv +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta +import io +import json +import os +import tempfile +from typing import Callable, Iterator, List, Optional, Sequence, Tuple +import unicodedata +import zipfile + + +TABULAR_MERGE_REPORT_VERSION = "tabular-merge-report-v1" + +SCHEMA_POLICY_BY_NAME = "by_name" +SCHEMA_POLICY_EXACT_ORDER = "exact_order" +TABULAR_MERGE_SCHEMA_POLICIES = (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER) + +TABULAR_MERGE_FORMAT_CSV = "csv" +TABULAR_MERGE_FORMAT_XLSX = "xlsx" +TABULAR_MERGE_FORMAT_XLS = "xls" +TABULAR_MERGE_EXTENSION_FORMATS = { + ".csv": TABULAR_MERGE_FORMAT_CSV, + ".xlsx": TABULAR_MERGE_FORMAT_XLSX, + ".xlsm": TABULAR_MERGE_FORMAT_XLSX, + ".xls": TABULAR_MERGE_FORMAT_XLS, +} +TABULAR_MERGE_SUPPORTED_EXTENSIONS = tuple(TABULAR_MERGE_EXTENSION_FORMATS) + +DEFAULT_SOURCE_COLUMN_NAME = "Source File" +MAX_SOURCE_COLUMN_NAME_CHARS = 128 +MAX_SHEET_NAME_CHARS = 31 + +SOURCE_STATUS_MERGED = "merged" +SOURCE_STATUS_COMPATIBLE = "compatible" +SOURCE_STATUS_SCHEMA_MISMATCH = "schema_mismatch" +SOURCE_STATUS_NOT_PROCESSED = "not_processed" + +_CSV_DELIMITERS = (",", ";", "\t", "|") +_CSV_SNIFF_BYTES = 64 * 1024 +_SPOOL_MEMORY_BYTES = 8 * 1024 * 1024 +_CANCEL_CHECK_INTERVAL = 1000 +_ZIP_MAGIC = b"PK\x03\x04" +_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +_MAX_EXACT_INTEGER_FLOAT = float(2 ** 53) + +_EXCEL_LIMITATIONS = ( + "Excel formulas are merged as their last calculated values; a workbook saved without " + "calculated values merges those cells as blanks.", + "Excel numbers, booleans and dates are written as plain text, with dates in ISO 8601 form.", +) + + +class TabularMergeError(ValueError): + """A safe, user-presentable merge failure with a stable code.""" + + def __init__(self, code, message, *, report=None, details=None): + super().__init__(message) + self.code = code + self.message = message + self.report = report + self.details = dict(details or {}) + + +class TabularMergeCancelled(TabularMergeError): + """The caller asked the merge to stop.""" + + def __init__(self): + super().__init__("cancelled", "The merge was cancelled.") + + +@dataclass(frozen=True) +class TabularMergeLimits: + """Bounds for one merge; every value must be a positive integer.""" + + max_sources: int = 10 + max_total_rows: int = 250_000 + max_source_bytes: int = 100 * 1024 * 1024 + max_columns: int = 256 + max_header_bytes: int = 256 + max_cell_chars: int = 32_767 + max_workbook_uncompressed_bytes: int = 512 * 1024 * 1024 + max_workbook_entries: int = 10_000 + + def __post_init__(self): + for name in ( + "max_sources", "max_total_rows", "max_source_bytes", "max_columns", + "max_header_bytes", "max_cell_chars", "max_workbook_uncompressed_bytes", + "max_workbook_entries", + ): + value = getattr(self, name) + if type(value) is not int or value < 1: + raise ValueError(f"{name} must be a positive integer.") + if self.max_sources < 2: + raise ValueError("max_sources must allow at least two sources.") + + +@dataclass(frozen=True) +class TabularMergeOptions: + """How sources are matched and how provenance is recorded.""" + + schema_policy: str = SCHEMA_POLICY_BY_NAME + sheet: Optional[str] = None + include_source_column: bool = True + source_column_name: str = DEFAULT_SOURCE_COLUMN_NAME + + def __post_init__(self): + if self.schema_policy not in TABULAR_MERGE_SCHEMA_POLICIES: + raise TabularMergeError( + "invalid_options", + f"Schema policy must be one of: {', '.join(TABULAR_MERGE_SCHEMA_POLICIES)}.", + ) + if self.sheet is not None: + if not isinstance(self.sheet, str) or not self.sheet.strip(): + raise TabularMergeError("invalid_options", "A sheet name must be nonempty text.") + if len(self.sheet) > MAX_SHEET_NAME_CHARS: + raise TabularMergeError( + "invalid_options", f"A sheet name has at most {MAX_SHEET_NAME_CHARS} characters.", + ) + if type(self.include_source_column) is not bool: + raise TabularMergeError("invalid_options", "include_source_column must be true or false.") + name = _display_header(self.source_column_name) if isinstance(self.source_column_name, str) else "" + if not name or len(name) > MAX_SOURCE_COLUMN_NAME_CHARS or len(name.encode("utf-8")) > 240: + raise TabularMergeError( + "invalid_options", + f"The source column name must be 1 to {MAX_SOURCE_COLUMN_NAME_CHARS} characters.", + ) + + +@dataclass(frozen=True) +class TabularMergeSource: + """One authorized source. ``load_bytes`` returns the original file bytes.""" + + source_id: str + file_name: str + load_bytes: Callable[[], bytes] = field(repr=False) + extension: str = "" + + def resolved_extension(self): + extension = (self.extension or os.path.splitext(self.file_name or "")[1]).strip().lower() + if extension and not extension.startswith("."): + extension = f".{extension}" + return extension + + +def tabular_merge_format_for_file_name(file_name): + """The merge reader format for a file name, or None when unsupported.""" + extension = os.path.splitext(str(file_name or ""))[1].strip().lower() + return TABULAR_MERGE_EXTENSION_FORMATS.get(extension) + + +def iter_csv_rows(content, file_name="A selected file"): + """Decode CSV bytes; return ``(encoding, delimiter, rows)`` where rows yields lists of text.""" + source = TabularMergeSource(file_name, file_name, lambda: content) + text, encoding = _decode_csv(content, source) + delimiter = _detect_delimiter(text) + reader = csv.reader( + io.StringIO(text, newline=""), delimiter=delimiter, quotechar='"', doublequote=True, strict=False, + ) + + def rows(): + try: + for values in reader: + yield values + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} couldn't be read as CSV near line {reader.line_num}.", + ) from exc + + return encoding, delimiter, rows() + + +def normalize_header_name(value): + """The comparison key for a header: NFC, collapsed whitespace, casefolded.""" + return _display_header(value).casefold() + + +def _display_header(value): + text = unicodedata.normalize("NFC", str(value if value is not None else "")) + return " ".join(text.split()) + + +class _RowSpool: + """Validated rows as compact JSON lines; small merges stay in memory.""" + + def __init__(self): + self._file = tempfile.SpooledTemporaryFile(max_size=_SPOOL_MEMORY_BYTES, mode="w+b") + self.count = 0 + self.closed = False + + def append(self, values): + self._file.write(json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + self._file.write(b"\n") + self.count += 1 + + def iter_rows(self): + if self.closed: + raise TabularMergeError("result_closed", "The merged rows are no longer available.") + self._file.flush() + self._file.seek(0) + yielded = 0 + for line in self._file: + if not line.strip(): + continue + yielded += 1 + yield json.loads(line) + if yielded != self.count: + raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") + + def close(self): + if not self.closed: + self.closed = True + self._file.close() + + +class TabularMergeResult: + """The merged table: ordered columns, an exact row count, rows and a report. + + Rows can be iterated more than once until ``close()``; each pass verifies the + count. Use it as a context manager so the spooled rows are always released. + """ + + def __init__(self, columns, report, spool): + self._columns = tuple(columns) + self._report = report + self._spool = spool + + @property + def columns(self): + return self._columns + + @property + def row_count(self): + return self._spool.count + + @property + def report(self): + return json.loads(json.dumps(self._report)) + + def iter_rows(self): + return self._spool.iter_rows() + + def iter_records(self): + columns = self._columns + for values in self._spool.iter_rows(): + yield dict(zip(columns, values)) + + def close(self): + self._spool.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + self.close() + return False + + +def merge_tabular_sources( + sources: Sequence[TabularMergeSource], + *, + options: Optional[TabularMergeOptions] = None, + limits: Optional[TabularMergeLimits] = None, + cancel_requested: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[dict], None]] = None, +) -> TabularMergeResult: + """Append the rows of same-structure sources, in the given order, into one table. + + The first source defines the column order. ``by_name`` accepts the same columns in + any order after header normalization; ``exact_order`` also requires the same order. + A schema mismatch reads every remaining header so the report lists all mismatches. + """ + options = options or TabularMergeOptions() + limits = limits or TabularMergeLimits() + sources = list(sources or ()) + _validate_sources(sources, limits) + + report = _new_report(options, len(sources)) + spool = _RowSpool() + reference = None + output_columns = () + source_column = None + mismatch_found = False + blank_rows_total = 0 + try: + for index, source in enumerate(sources): + _check_cancel(cancel_requested) + entry = _new_source_entry(source) + report["sources"].append(entry) + content = _load_source_bytes(source, limits) + table = _open_table(source, content, options, limits) + try: + entry.update(table.describe()) + header = _header_names(table.header, source, limits, entry) + if reference is None: + reference = header + source_column = _source_column_name(options, header, report) + output_columns = ((source_column,) if source_column else ()) + tuple( + display for display, _ in header + ) + if len(output_columns) > limits.max_columns: + raise TabularMergeError( + "too_many_columns", + f"The merged table would have {len(output_columns)} columns; " + f"at most {limits.max_columns} are supported.", + ) + report["columns"] = list(output_columns) + report["source_column"] = source_column + column_map = list(range(len(header))) + else: + column_map = _match_header(reference, header, options.schema_policy, entry) + if column_map is None: + mismatch_found = True + entry["status"] = SOURCE_STATUS_SCHEMA_MISMATCH + continue + + if mismatch_found: + entry["status"] = SOURCE_STATUS_NOT_PROCESSED + continue + + rows, blank_rows = _append_rows( + table, source, column_map, len(header), source_column, spool, limits, + cancel_requested, entry, + ) + blank_rows_total += blank_rows + entry["rows"] = rows + entry["blank_rows_skipped"] = blank_rows + entry["status"] = SOURCE_STATUS_MERGED + finally: + table.close() + if on_progress is not None: + on_progress({ + "index": index + 1, "total": len(sources), + "file_name": source.file_name, "rows": entry.get("rows", 0), + }) + + if mismatch_found: + report["status"] = "failed" + for entry in report["sources"]: + if entry["status"] in (SOURCE_STATUS_MERGED, SOURCE_STATUS_NOT_PROCESSED): + entry["status"] = SOURCE_STATUS_COMPATIBLE + entry["rows"] = 0 + entry["blank_rows_skipped"] = 0 + entry["short_rows_padded"] = 0 + raise TabularMergeError( + "schema_mismatch", _mismatch_message(report), report=report, + ) + + report["status"] = "merged" + report["totals"] = { + "sources": len(sources), + "rows": spool.count, + "columns": len(output_columns), + "blank_rows_skipped": blank_rows_total, + } + if any(entry.get("format") != TABULAR_MERGE_FORMAT_CSV for entry in report["sources"]): + report["limitations"] = list(_EXCEL_LIMITATIONS) + return TabularMergeResult(output_columns, report, spool) + except BaseException: + spool.close() + raise + + +def _validate_sources(sources, limits): + if len(sources) < 2: + raise TabularMergeError("too_few_sources", "Merging needs at least two files.") + if len(sources) > limits.max_sources: + raise TabularMergeError( + "too_many_sources", + f"{len(sources)} files were selected; at most {limits.max_sources} can be merged here.", + ) + seen = set() + for source in sources: + if not isinstance(source, TabularMergeSource): + raise TabularMergeError("invalid_source", "Each merge source must be a TabularMergeSource.") + if not isinstance(source.source_id, str) or not source.source_id.strip(): + raise TabularMergeError("invalid_source", "Each merge source needs an identifier.") + if source.source_id in seen: + raise TabularMergeError("duplicate_source", "The same file was selected more than once.") + seen.add(source.source_id) + if not callable(source.load_bytes): + raise TabularMergeError("invalid_source", "Each merge source needs a byte loader.") + if TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()) is None: + raise TabularMergeError( + "unsupported_format", + f"{_safe_name(source)} isn't a CSV or Excel file (.csv, .xlsx, .xlsm, .xls).", + ) + + +def _new_report(options, source_count): + return { + "version": TABULAR_MERGE_REPORT_VERSION, + "status": "running", + "policy": { + "schema_policy": options.schema_policy, + "sheet": options.sheet, + "include_source_column": options.include_source_column, + "source_column_name": _display_header(options.source_column_name) if options.include_source_column else None, + }, + "columns": [], + "source_column": None, + "totals": {"sources": source_count, "rows": 0, "columns": 0, "blank_rows_skipped": 0}, + "sources": [], + "warnings": [], + "limitations": [], + } + + +def _new_source_entry(source): + return { + "source_id": source.source_id, + "file_name": _safe_name(source), + "format": TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()), + "sheet": None, + "encoding": None, + "delimiter": None, + "rows": 0, + "blank_rows_skipped": 0, + "short_rows_padded": 0, + "status": SOURCE_STATUS_NOT_PROCESSED, + "missing_columns": [], + "extra_columns": [], + "order_differs": False, + "warnings": [], + } + + +def _safe_name(source): + name = str(getattr(source, "file_name", "") or "").strip() + return name or "A selected file" + + +def _check_cancel(cancel_requested): + if cancel_requested is not None and cancel_requested(): + raise TabularMergeCancelled() + + +def _load_source_bytes(source, limits): + content = source.load_bytes() + if isinstance(content, bytearray): + content = bytes(content) + if not isinstance(content, bytes): + raise TabularMergeError("source_unavailable", f"{_safe_name(source)} could not be read.") + if len(content) > limits.max_source_bytes: + raise TabularMergeError( + "source_too_large", + f"{_safe_name(source)} is larger than the {limits.max_source_bytes // (1024 * 1024)} MB merge limit.", + ) + return content + + +def _source_column_name(options, header, report): + if not options.include_source_column: + return None + base = _display_header(options.source_column_name) + taken = {normalized for _, normalized in header} + candidate = base + suffix = 2 + while candidate.casefold() in taken: + candidate = f"{base} ({suffix})" + suffix += 1 + if candidate != base: + report["warnings"].append({ + "code": "source_column_renamed", + "message": f'The files already have a "{base}" column, so the file name column is "{candidate}".', + }) + return candidate + + +def _header_names(raw_header, source, limits, entry): + """Display and comparison names for one header row; trailing blanks are dropped.""" + cells = list(raw_header) + while cells and not _display_header(cells[-1]): + cells.pop() + if not cells: + raise TabularMergeError("empty_source", f"{_safe_name(source)} has no header row.") + names = [] + seen = {} + for position, value in enumerate(cells, start=1): + display = _display_header(value) + if not display: + display = f"Column {position}" + entry["warnings"].append({ + "code": "blank_header", + "message": f'Column {position} has no header, so it is named "{display}".', + }) + if len(display.encode("utf-8")) > limits.max_header_bytes: + raise TabularMergeError( + "header_too_long", + f"A column header in {_safe_name(source)} is longer than {limits.max_header_bytes} bytes.", + ) + normalized = display.casefold() + if normalized in seen: + raise TabularMergeError( + "duplicate_columns", + f'{_safe_name(source)} has more than one "{display}" column.', + ) + seen[normalized] = position + names.append((display, normalized)) + return names + + +def _match_header(reference, header, schema_policy, entry): + """Positions in ``header`` for each reference column, or None on a mismatch.""" + reference_keys = [normalized for _, normalized in reference] + header_keys = [normalized for _, normalized in header] + positions = {key: index for index, key in enumerate(header_keys)} + missing = [display for display, key in reference if key not in positions] + reference_set = set(reference_keys) + extra = [display for display, key in header if key not in reference_set] + entry["missing_columns"] = missing + entry["extra_columns"] = extra + if missing or extra: + return None + if reference_keys != header_keys: + entry["order_differs"] = True + if schema_policy == SCHEMA_POLICY_EXACT_ORDER: + return None + return [positions[key] for key in reference_keys] + + +def _mismatch_message(report): + reference = report["sources"][0]["file_name"] if report["sources"] else "the first file" + parts = [] + for entry in report["sources"]: + if entry["status"] != SOURCE_STATUS_SCHEMA_MISMATCH: + continue + details = [] + if entry["missing_columns"]: + details.append(f"missing {', '.join(_quoted(entry['missing_columns']))}") + if entry["extra_columns"]: + details.append(f"extra {', '.join(_quoted(entry['extra_columns']))}") + if not details and entry["order_differs"]: + details.append("columns in a different order") + parts.append(f"{entry['file_name']} ({'; '.join(details)})") + return ( + f"These files don't have the same columns as {reference}: {'; '.join(parts)}. " + "Nothing was merged." + ) + + +def _quoted(values, limit=8): + shown = [f'"{value}"' for value in values[:limit]] + if len(values) > limit: + shown.append(f"and {len(values) - limit} more") + return shown + + +def _append_rows(table, source, column_map, width, source_column, spool, limits, cancel_requested, entry): + rows = 0 + blank_rows = 0 + for row_number, values in table.rows(): + if rows % _CANCEL_CHECK_INTERVAL == 0: + _check_cancel(cancel_requested) + cells = list(values) + if len(cells) > width: + if any(_cell_has_content(value) for value in cells[width:]): + raise TabularMergeError( + "row_has_extra_values", + f"Row {row_number} in {_safe_name(source)} has more values than there are columns.", + ) + cells = cells[:width] + if not any(_cell_has_content(value) for value in cells): + blank_rows += 1 + continue + if len(cells) < width: + entry["short_rows_padded"] += 1 + cells.extend([""] * (width - len(cells))) + for value in cells: + if len(value) > limits.max_cell_chars: + raise TabularMergeError( + "cell_too_large", + f"Row {row_number} in {_safe_name(source)} has a value longer than " + f"{limits.max_cell_chars} characters.", + ) + if spool.count >= limits.max_total_rows: + raise TabularMergeError( + "row_limit_exceeded", + f"The merged table would have more than {limits.max_total_rows:,} rows. " + "Merge fewer files here, or use a workflow for larger merges.", + ) + ordered = [cells[position] for position in column_map] + spool.append(([source.file_name] if source_column else []) + ordered) + rows += 1 + return rows, blank_rows + + +def _cell_has_content(value): + return bool(value) and bool(str(value).strip()) + + +# --------------------------------------------------------------------------- +# Table readers +# --------------------------------------------------------------------------- + + +class _Table: + """A header plus a row iterator over one sheet or CSV body.""" + + format = "" + + def describe(self): + return {} + + @property + def header(self): + raise NotImplementedError + + def rows(self): + raise NotImplementedError + + def close(self): + return None + + +def _open_table(source, content, options, limits): + table_format = TABULAR_MERGE_EXTENSION_FORMATS[source.resolved_extension()] + if table_format == TABULAR_MERGE_FORMAT_CSV: + return _CsvTable(source, content) + if table_format == TABULAR_MERGE_FORMAT_XLSX: + return _XlsxTable(source, content, options.sheet, limits) + return _XlsTable(source, content, options.sheet) + + +class _CsvTable(_Table): + format = TABULAR_MERGE_FORMAT_CSV + + def __init__(self, source, content): + self._source = source + text, self._encoding = _decode_csv(content, source) + self._delimiter = _detect_delimiter(text) + self._reader = csv.reader( + io.StringIO(text, newline=""), delimiter=self._delimiter, quotechar='"', + doublequote=True, strict=False, + ) + self._header = None + self._header_line = 0 + try: + for values in self._reader: + if any(_cell_has_content(value) for value in values): + self._header = values + self._header_line = self._reader.line_num + break + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} couldn't be read as CSV near line {self._reader.line_num}.", + ) from exc + if self._header is None: + raise TabularMergeError("empty_source", f"{_safe_name(source)} has no header row.") + + def describe(self): + return {"encoding": self._encoding, "delimiter": self._delimiter} + + @property + def header(self): + return self._header + + def rows(self): + reader = self._reader + try: + for values in reader: + yield reader.line_num, values + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(self._source)} couldn't be read as CSV near line {reader.line_num}.", + ) from exc + + +def _decode_csv(content, source): + for bom, encoding in ( + (codecs.BOM_UTF32_LE, "utf-32"), (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF8, "utf-8-sig"), + (codecs.BOM_UTF16_LE, "utf-16"), (codecs.BOM_UTF16_BE, "utf-16"), + ): + if content.startswith(bom): + try: + return content.decode(encoding), encoding + except UnicodeDecodeError as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} has text that doesn't match its encoding.", + ) from exc + if b"\x00" in content[:_CSV_SNIFF_BYTES]: + raise TabularMergeError("unreadable_csv", f"{_safe_name(source)} doesn't look like a text CSV file.") + for encoding in ("utf-8", "cp1252"): + try: + return content.decode(encoding), encoding + except UnicodeDecodeError: + continue + return content.decode("latin-1"), "latin-1" + + +def _detect_delimiter(text): + """The most frequent supported delimiter outside quotes on the first nonblank line.""" + counts = {delimiter: 0 for delimiter in _CSV_DELIMITERS} + in_quotes = False + seen_content = False + for character in text[:_CSV_SNIFF_BYTES]: + if character == '"': + in_quotes = not in_quotes + seen_content = True + continue + if in_quotes: + continue + if character in "\r\n": + if seen_content: + break + continue + if character in counts: + counts[character] += 1 + if not character.isspace(): + seen_content = True + best = max(_CSV_DELIMITERS, key=lambda delimiter: (counts[delimiter], -_CSV_DELIMITERS.index(delimiter))) + return best if counts[best] else "," + + +class _XlsxTable(_Table): + format = TABULAR_MERGE_FORMAT_XLSX + + def __init__(self, source, content, sheet, limits): + self._source = source + _guard_workbook_archive(content, source, limits) + # openpyxl is imported only when a workbook is merged; it is a large binary dependency. + from openpyxl import load_workbook + + try: + self._workbook = load_workbook( + io.BytesIO(content), read_only=True, data_only=True, keep_links=False, + ) + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} couldn't be opened as an Excel workbook.", + ) from exc + try: + self._sheet = _select_sheet( + [(worksheet.title, getattr(worksheet, "sheet_state", "visible")) for worksheet in self._workbook.worksheets], + sheet, source, + ) + self._worksheet = self._workbook[self._sheet] + self._worksheet.reset_dimensions() + self._iterator = self._worksheet.iter_rows(values_only=True) + self._row_number = 0 + self._header = None + for values in self._iterator: + self._row_number += 1 + cells = [_excel_text(value) for value in values] + if any(_cell_has_content(value) for value in cells): + self._header = cells + break + except TabularMergeError: + self.close() + raise + except Exception as exc: + self.close() + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} couldn't be read as an Excel workbook.", + ) from exc + if self._header is None: + self.close() + raise TabularMergeError( + "empty_source", f'Sheet "{self._sheet}" in {_safe_name(source)} has no header row.', + ) + + def describe(self): + return {"sheet": self._sheet} + + @property + def header(self): + return self._header + + def rows(self): + try: + for values in self._iterator: + self._row_number += 1 + yield self._row_number, [_excel_text(value) for value in values] + except TabularMergeError: + raise + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self._source)} couldn't be read near row {self._row_number}.", + ) from exc + + def close(self): + workbook = getattr(self, "_workbook", None) + if workbook is not None: + self._workbook = None + try: + workbook.close() + except Exception: + pass + + +def _guard_workbook_archive(content, source, limits): + if content[:8] == _OLE_MAGIC: + raise TabularMergeError( + "encrypted_workbook", + f"{_safe_name(source)} is password-protected or encrypted, so it can't be merged.", + ) + if content[:4] != _ZIP_MAGIC: + raise TabularMergeError("unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel workbook.") + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + entries = archive.infolist() + except zipfile.BadZipFile as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel workbook.", + ) from exc + if len(entries) > limits.max_workbook_entries: + raise TabularMergeError("workbook_too_large", f"{_safe_name(source)} has too many internal parts to merge.") + if sum(entry.file_size for entry in entries) > limits.max_workbook_uncompressed_bytes: + raise TabularMergeError("workbook_too_large", f"{_safe_name(source)} is too large to merge once uncompressed.") + + +class _XlsTable(_Table): + format = TABULAR_MERGE_FORMAT_XLS + + def __init__(self, source, content, sheet): + self._source = source + if content[:8] != _OLE_MAGIC: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel 97-2003 (.xls) workbook.", + ) + # xlrd is imported only when a legacy workbook is merged. + import xlrd + + self._xlrd = xlrd + try: + self._book = xlrd.open_workbook(file_contents=content, on_demand=True) + except Exception as exc: + code = "encrypted_workbook" if "encrypt" in str(exc).lower() else "unreadable_workbook" + message = ( + f"{_safe_name(source)} is password-protected or encrypted, so it can't be merged." + if code == "encrypted_workbook" + else f"{_safe_name(source)} couldn't be opened as an Excel workbook." + ) + raise TabularMergeError(code, message) from exc + try: + names = self._book.sheet_names() + states = [] + for index, name in enumerate(names): + visibility = 0 + if sheet is None: + visibility = getattr(self._book.sheet_by_index(index), "visibility", 0) + states.append((name, "visible" if visibility == 0 else "hidden")) + self._sheet = _select_sheet(states, sheet, source) + self._worksheet = self._book.sheet_by_name(self._sheet) + self._datemode = self._book.datemode + self._next_row = 0 + self._header = None + while self._next_row < self._worksheet.nrows: + cells = self._row_values(self._next_row) + self._next_row += 1 + if any(_cell_has_content(value) for value in cells): + self._header = cells + break + except TabularMergeError: + self.close() + raise + except Exception as exc: + self.close() + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} couldn't be read as an Excel workbook.", + ) from exc + if self._header is None: + self.close() + raise TabularMergeError( + "empty_source", f'Sheet "{self._sheet}" in {_safe_name(source)} has no header row.', + ) + + def describe(self): + return {"sheet": self._sheet} + + @property + def header(self): + return self._header + + def rows(self): + while self._next_row < self._worksheet.nrows: + row_number = self._next_row + 1 + cells = self._row_values(self._next_row) + self._next_row += 1 + yield row_number, cells + + def _row_values(self, row_index): + xlrd = self._xlrd + values = [] + for cell in self._worksheet.row(row_index): + if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + values.append("") + elif cell.ctype == xlrd.XL_CELL_DATE: + try: + values.append(_excel_text(xlrd.xldate_as_datetime(cell.value, self._datemode))) + except Exception: + values.append(_excel_text(cell.value)) + elif cell.ctype == xlrd.XL_CELL_BOOLEAN: + values.append("TRUE" if cell.value else "FALSE") + elif cell.ctype == xlrd.XL_CELL_ERROR: + values.append(xlrd.error_text_from_code.get(cell.value, "#ERROR")) + else: + values.append(_excel_text(cell.value)) + return values + + def close(self): + book = getattr(self, "_book", None) + if book is not None: + self._book = None + try: + book.release_resources() + except Exception: + pass + + +def _select_sheet(sheets: List[Tuple[str, str]], requested, source): + names = [name for name, _ in sheets] + if not names: + raise TabularMergeError("empty_source", f"{_safe_name(source)} has no worksheets.") + if requested is not None: + if requested in names: + return requested + folded = requested.casefold() + for name in names: + if name.casefold() == folded: + return name + raise TabularMergeError( + "sheet_not_found", + f'{_safe_name(source)} has no sheet named "{requested}". Its sheets are: ' + f"{', '.join(_quoted(names))}.", + details={"available_sheets": names}, + ) + for name, state in sheets: + if state == "visible": + return name + return names[0] + + +def _excel_text(value): + """Canonical text for one Excel cell value.""" + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer() and abs(value) < _MAX_EXACT_INTEGER_FLOAT: + return str(int(value)) + return repr(value) + if isinstance(value, datetime): + if value.tzinfo is None and value.time() == time(0, 0): + return value.date().isoformat() + return value.isoformat(timespec="seconds" if value.microsecond == 0 else "microseconds") + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat(timespec="seconds" if value.microsecond == 0 else "microseconds") + if isinstance(value, timedelta): + total_seconds = int(value.total_seconds()) + sign = "-" if total_seconds < 0 else "" + hours, remainder = divmod(abs(total_seconds), 3600) + minutes, seconds = divmod(remainder, 60) + return f"{sign}{hours}:{minutes:02d}:{seconds:02d}" + return str(value) + + +__all__ = [ + "DEFAULT_SOURCE_COLUMN_NAME", + "SCHEMA_POLICY_BY_NAME", + "SCHEMA_POLICY_EXACT_ORDER", + "TABULAR_MERGE_REPORT_VERSION", + "TABULAR_MERGE_SCHEMA_POLICIES", + "TABULAR_MERGE_SUPPORTED_EXTENSIONS", + "TabularMergeCancelled", + "TabularMergeError", + "TabularMergeLimits", + "TabularMergeOptions", + "TabularMergeResult", + "TabularMergeSource", + "iter_csv_rows", + "merge_tabular_sources", + "normalize_header_name", + "tabular_merge_format_for_file_name", +] diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index e45089347..8b489800a 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1703,6 +1703,33 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul 10, ), }, + 'merge': { + 'enabled': form_data.get('document_action_merge_enabled') == 'on', + 'chat_max_documents': parse_admin_int( + form_data.get('document_action_merge_chat_max_documents'), + current_document_action_capabilities.get('merge', {}).get('chat_max_documents', 10), + 'document_action_merge_chat_max_documents', + 10, + ), + 'workflow_max_documents': parse_admin_int( + form_data.get('document_action_merge_workflow_max_documents'), + current_document_action_capabilities.get('merge', {}).get('workflow_max_documents', 100), + 'document_action_merge_workflow_max_documents', + 100, + ), + 'chat_max_rows': parse_admin_int( + form_data.get('document_action_merge_chat_max_rows'), + current_document_action_capabilities.get('merge', {}).get('chat_max_rows', 250000), + 'document_action_merge_chat_max_rows', + 250000, + ), + 'workflow_max_rows': parse_admin_int( + form_data.get('document_action_merge_workflow_max_rows'), + current_document_action_capabilities.get('merge', {}).get('workflow_max_rows', 1000000), + 'document_action_merge_workflow_max_rows', + 1000000, + ), + }, } }) diff --git a/application/single_app/templates/admin/_panes/actions.html b/application/single_app/templates/admin/_panes/actions.html index 8030031af..bd747b705 100644 --- a/application/single_app/templates/admin/_panes/actions.html +++ b/application/single_app/templates/admin/_panes/actions.html @@ -4,6 +4,7 @@ set in one pane is not visible to the next one. #} {% set analyze_capability = settings.document_action_capabilities.analyze %} {% set comparison_capability = settings.document_action_capabilities.comparison %} + {% set merge_capability = settings.document_action_capabilities.merge or {} %}
Microsoft 365 retrieval

Auto uses Copilot Retrieval only for verified licensed users in supported clouds. Other users use delegated Microsoft Graph. Pay-as-you-go is not used.

@@ -25,7 +26,7 @@
Document Action CapabilitiesChat: 2-300 | Workflow: 2-1000
@@ -118,6 +119,89 @@
Document Action Capabilities
+
+
+
+ + + +
+
+
+
+ + {{ merge_capability.chat_max_documents|default(10) }} +
+
+
+ +
+
+ +
+
+
+ 2 + 300 +
+
+
+
+ + {{ merge_capability.workflow_max_documents|default(100) }} +
+
+
+ +
+
+ +
+
+
+ 2 + 1000 +
+
+
+
+ + {{ merge_capability.chat_max_rows|default(250000) }} +
+
+
+ +
+
+ +
+
+
+ 1,000 + 1,000,000 +
+
+
+
+ + {{ merge_capability.workflow_max_rows|default(1000000) }} +
+
+
+ +
+
+ +
+
+
+ 1,000 + 1,000,000 +
+
+
+
+
diff --git a/docs/admin/agents-actions.md b/docs/admin/agents-actions.md index 0a0c65851..8b9678edc 100644 --- a/docs/admin/agents-actions.md +++ b/docs/admin/agents-actions.md @@ -201,12 +201,20 @@ full, so an answer covers all of them rather than only the passages a search returned; Comparison reads one baseline document against the others, which is what answers questions about what changed between versions. +**Merge** combines the rows of several CSV or Excel files that share the same +columns into one table. Since **0.261.218**, V2 chat orchestration uses it when +someone asks to merge, combine or append spreadsheets, and the plan delivers the +result as a CSV or Excel file. Code appends the rows; no model reads them, so a +merge costs far less than analyzing the same files. That is why its limits are +higher, and why it is limited by merged rows as well as by files: a few very +large files can cost more than many small ones. + Because each document is read in full, the limits are the control that matters. They bound how long a single message can take and how much it costs. Chat and workflow are limited separately: a chat message has someone waiting on it, while a workflow run does not and can be allowed a much larger batch. -These six values are stored as one object, `document_action_capabilities`, rather +These values are stored as one object, `document_action_capabilities`, rather than as separate settings. #### Settings @@ -219,6 +227,11 @@ than as separate settings. | Enable Document Comparison | Offers Document Comparison in the Action menu. | On | — | `document_action_capabilities.comparison.enabled` | | Comparison: Chat Document Limit | Most documents one chat message may compare, including the baseline. | 3 | 2–300 | `comparison.chat_max_documents` | | Comparison: Workflow Document Limit | The same limit for a workflow run. | 10 | 2–1000 | `comparison.workflow_max_documents` | +| Enable Merge | Lets chat orchestration combine same-structure CSV and Excel files into one file. | On | — | `document_action_capabilities.merge.enabled`; also governed by the **Merge spreadsheets** orchestration capability. | +| Merge: Chat Document Limit | Most files one chat request may merge. | 10 | 2–300 | `merge.chat_max_documents` | +| Merge: Workflow Document Limit | The same limit for a workflow run. | 100 | 2–1000 | `merge.workflow_max_documents` | +| Merge: Chat Row Limit | Most rows one chat merge may produce. A merge over the limit stops without creating a file. | 250,000 | 1,000–1,000,000 | `merge.chat_max_rows` | +| Merge: Workflow Row Limit | The same row limit for a workflow run. | 1,000,000 | 1,000–1,000,000 | `merge.workflow_max_rows` | Values outside the range are clamped on save rather than rejected. diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index 887c8c8d0..eafc6b4d1 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -228,6 +228,20 @@ use personal workflows, in their own private conversations. Reading a result changes nothing, so it never makes a plan wait for approval. Clearing it keeps saved results out of plans; **Ask in chat** on a run is not affected. +Since **0.261.218**, **Merge spreadsheets** (`tabular_merge`) combines the rows +of two or more CSV or Excel files that share the same columns into one table, +which the plan then delivers as a CSV or Excel file through **Create a file**. +Code appends every row exactly, in the order the files are listed, and adds a +**Source File** column by default; no model reads or rewrites the rows, so +codes such as `007` keep their leading zeros. Columns may appear in any order. +Files whose columns differ are not merged: the run says so and nothing is +created. Merge never matches rows on a key column the way a lookup or join +would. It's offered while **Enable Merge** is on under +[Document Action Capabilities]({{ '/admin/agents-actions/' | relative_url }}#document-action-capabilities-card), +which also sets how many files and rows one chat request may merge. Clearing +it here keeps merges out of plans. See +[Merge files in chat]({{ '/guides/merge-files/' | relative_url }}). + Selecting a capability does not grant source/model access or enable another feature's prerequisite. Orchestration still publishes truthful delivery status when composition is not requested, including file-only plans that consume already diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index 28c332e9d..f088e2ef1 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -130,6 +130,7 @@ permissions cannot be supplied as bindings. | `document_analyze` | Narrative `findings: records-v1` and `coverage: structured-v1`; public `records` and a Markdown `report` are optional outputs only when genuinely produced. | | `document_compare` | Narrative `comparison: comparison-v1` and `coverage: structured-v1`; a Markdown `report` is optional when available. | | `tabular_analyze` | Server-bound native query/transform returns `records` and `coverage`; analysis-only returns `analysis` and `coverage`; combined transformation/analysis returns all three. | +| `tabular_merge` | Since **0.261.218**: `records: records-v1` holding every row of two or more same-structure CSV or Excel sources, appended by code in source order, and `report: structured-v1` (`tabular-merge-report-v1`) with the policy, columns, per-source rows and limitations. Coverage counts sources. See [V2 file merge](V2_FILE_MERGE.md). | | `compose` | Explicitly named text, Markdown, records or structured values. There is no default output declaration. | | Server-bound external Gather | `prepared: structured-v1` with exact returned content and separately authorized external-source lineage. | diff --git a/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md b/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md index 682e28c12..070f93c6d 100644 --- a/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md +++ b/docs/explanation/features/GENERATED_FILE_EXPORT_FRAMEWORK.md @@ -2,7 +2,7 @@ Implemented in version: **0.250.072** -Updated through version: **0.261.127** +Updated through version: **0.261.218** GitHub issue: [#1071](https://github.com/microsoft/simplechat/issues/1071) @@ -152,10 +152,12 @@ orchestration capability. The workflow admission constant | `json`, `yaml` / `yml` | `structured_records_v1` | `records` | Complete array/sequence, record and object insertion order, nested values and JSON-domain types preserved. | | `json`, `yaml` / `yml` | `structured_value_v1` | `structured_value` | One complete JSON-domain value, preserving object insertion order. | | `csv` | `tabular_records_v1` | `records` | Explicit ordered columns, matching scalar-only records and formula-safe headers/cells. | +| `csv` | `exact_tabular_records_v1` | `records` | Since **0.261.218**: every retained column in retained order, with the same scalar, formula-safe cells as `tabular_records_v1`. No options. Requires a source with a retained column schema. | | `xml` | `typed_xml_v1` | `records`, `structured_value` | A fixed typed XML vocabulary that preserves keys, nested values and scalar distinctions. | | `md` / `markdown` | `prepared_text_v1` | `markdown` | Prepared Markdown bytes; no hidden composition or rich-document conversion. | | `txt` / `text` | `prepared_text_v1` | `text` | Prepared plain text bytes. | | `xlsx` | `tabular_workbook_v1` | `records` | One explicitly named sheet, ordered columns and streaming scalar records. Required options: `columns`, `sheet_name`. | +| `xlsx` | `exact_tabular_workbook_v1` | `records` | Since **0.261.218**: one sheet holding every retained column in retained order. Optional `sheet_name`, default `Sheet1`. | | `docx`, `pdf` | `prepared_report_v1` | `text`, `markdown` | The same complete prepared report, with literal text or supported Markdown semantics according to source kind. Optional `title`. | | `pptx` | `prepared_slide_deck_v1` | `structured_value` | Versioned prepared-slide JSON with positioned shapes. No request options; deck title/size belong to the prepared JSON. | @@ -177,6 +179,16 @@ CSV is an explicit presentation profile, not a lossless typed interchange. Every row must contain exactly the requested columns; missing/extra fields and nested cells fail instead of being silently omitted or flattened. Columns must be nonempty and have distinct case-insensitive, formula-safe headers. + +The exact tabular profiles exist for results whose columns are decided at run +time, such as a merge of files found by a search: the plan cannot name columns +it has not seen. `exact_tabular_records_v1` and `exact_tabular_workbook_v1` +take the ordered column list from the source's retained schema +(`source.columns`) instead of the request. The projection is the identity, so +nothing is invented, reordered or dropped; a request that also names +`columns` is rejected, and a source without a retained schema fails as +`unsupported_source`. Every other rule of the matching explicit profile still +applies. Declared spelling, whitespace and order are preserved apart from necessary formula neutralization. Null becomes an empty cell, booleans become `true` or `false`, and numbers retain JSON numeric spelling. Strings retain their diff --git a/docs/explanation/features/V2_FILE_MERGE.md b/docs/explanation/features/V2_FILE_MERGE.md new file mode 100644 index 000000000..16afc2265 --- /dev/null +++ b/docs/explanation/features/V2_FILE_MERGE.md @@ -0,0 +1,132 @@ +# V2 File Merge + +Version: **0.261.218** + +Implemented in version: **0.261.218** (Phase 1, same-structure spreadsheets), recorded in +`application/single_app/config.py`. + +GitHub issue: [#1619](https://github.com/microsoft/simplechat/issues/1619) + +Related: [#1509](https://github.com/microsoft/simplechat/issues/1509) (Gather / Reason / +Render), [#1071](https://github.com/microsoft/simplechat/issues/1071) (generated file exports). + +## Overview + +File merge combines several files into one. People receive the same report from many +places — one export per region, month or team — and need a single file. Before this +feature SimpleChat could not do it: native tabular analysis accepts exactly one source, +the tabular stack had no append operation, and asking a model to "write a combined CSV" +is slow, costly and can silently drop or alter rows. + +Merge is delivered in phases. Spreadsheets come first because they are the most common +request and the easiest to verify exactly; documents and decks follow. + +| Phase | Scope | Status | +| --- | --- | --- | +| 1 | Same-structure CSV and Excel merge in V2 chat (`tabular_merge`), Merge limits, exact-schema CSV/XLSX exports | Implemented in **0.261.218** — see [Phase 1](V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md) | +| 2 | Reconciling different structures: column inspection, union and mapped policies, sheet modes, duplicate removal, AI-suggested column mapping | Planned | +| 3 | Large and recurring merges (100+ files) as V2 workflow tasks | Planned | +| 4 | Document assembly foundation, PDF merge, and workbooks with one tab per file | Planned | +| 5 | Word merge, keeping each document's styles or the first document's | Planned | +| 6 | PowerPoint merge, keeping each deck's look or the first deck's theme | Planned | +| 7 | Scale and security hardening, final documentation | Planned | + +## Where merge fits in Gather / Reason / Render + +Merge is another capability alongside Analyze and Compare. It does not replace any of +them, and it is reached the same way: the user asks in a V2 chat and the orchestrator +plans the work. + +| Purpose | Role in a merge | +| --- | --- | +| Gather (optional) | `document_search` finds the files when the user describes them instead of selecting them. Its `sources` output binds into the merge. | +| Reason | `tabular_merge` reads the authorized files and appends their rows with code into one retained table. No model reads or rewrites rows. | +| Render | `render_file` turns the retained table into a CSV or Excel file. | + +Merging is Reason, not Render, because it computes a new dataset from sources; Render only +serializes a prepared result. Keeping the merged table as a retained result also lets one +merge feed several files — a CSV and an Excel copy — without merging twice. + +Saved memory does not store or merge files. The planner does read saved preferences, so a +saved instruction such as "always give me Excel" shapes merge requests without extra work. + +## Technical specifications + +### Components + +| File | Responsibility | +| --- | --- | +| `functions_tabular_merge.py` | Pure merge engine: CSV, XLSX/XLSM and XLS readers that keep values as text, header normalization, schema policies, provenance column, limits, cancellation, archive guards, and the merge report. No Flask, settings, storage or model imports. | +| `functions_orchestration_merge.py` | Orchestration glue: Merge limits from settings, the authorized-source manifest check, screening-aware byte loaders, persistence of the retained `records` and `report`, and the engine-to-failure-code map. | +| `functions_orchestration_adapters.py` | `run_tabular_merge`, registered in `ADAPTER_REGISTRY`. | +| `functions_orchestration_registry.py` | The `tabular_merge` capability descriptor and the `merge` document-action gate. | +| `functions_orchestration_schema.py` | Plan rules: explicit document IDs or a bound source set, tabular-only sources, the Merge chat limit, and the merge failure messages. | +| `functions_document_actions.py` | The `merge` document action: enablement, file limits and row limits for chat and workflows. | +| `functions_generated_export_registry.py`, `functions_structured_file_renderers.py`, `functions_generated_office_adapters.py` | The `exact_tabular_records_v1` (CSV) and `exact_tabular_workbook_v1` (XLSX) profiles. | + +### Result contract + +`tabular_merge` retains two outputs under contract `tabular-merge-v1`: + +- `records` (`records-v1`): one string column per merged column, in output order, with + every row. At most 256 columns, because retained record schemas allow 256. +- `report` (`structured-v1`): `tabular-merge-report-v1`, holding the policy, the output + columns, the provenance column name, totals, one entry per source (file name, format, + sheet, encoding, delimiter, rows, skipped blank rows, padded short rows, column + differences, warnings) and limitations. + +Coverage counts sources. The result is complete only when every source merged; a merge +either finishes or produces nothing. + +### Limits + +| Limit | Default | Where | +| --- | --- | --- | +| Files per chat merge | 10 (2–300) | `document_action_capabilities.merge.chat_max_documents` | +| Files per workflow merge | 100 (2–1000) | `document_action_capabilities.merge.workflow_max_documents` | +| Rows per chat merge | 250,000 (1,000–1,000,000) | `document_action_capabilities.merge.chat_max_rows` | +| Rows per workflow merge | 1,000,000 (1,000–1,000,000) | `document_action_capabilities.merge.workflow_max_rows` | +| Bytes per source file | 100 MB | Engine constant | +| Columns | 256 | Retained record schema limit | +| Header name | 256 UTF-8 bytes | Retained column name limit | +| Cell | 32,767 characters | Excel's cell limit | +| Workbook expansion | 512 MB uncompressed, 10,000 parts | Engine guard | + +Rendering keeps its own limits: CSV exports hold at most 1,000,000 records, and XLSX at +most 1,048,575 rows, 5,000,000 cells and 32 MiB. CSV suits the largest merges. + +### Security + +- Every source is authorized again when the run starts and when its bytes are read, through + `content_screening.access.read_available_document_bytes`, which enforces document + access, screening holds and the source revision the plan approved. +- Workbooks are opened read-only with external links disabled; `defusedxml` protects XML + parsing. Encrypted workbooks are refused, and archive size is checked before parsing. +- Values stay text and are never evaluated. CSV output neutralizes formula-like cells; + XLSX output writes them as literal text. +- Failure messages are application-owned text. File names and header text from users' + files never appear in a failure message. + +## Usage + +Users ask in a V2 chat, for example "merge these three regional CSVs into one Excel file". +See [Merge files in chat](../../guides/merge-files.md). Administrators control the feature +under [Document Action Capabilities](../../admin/agents-actions.md#document-action-capabilities-card) +and the orchestration Capabilities list +([orchestration settings](../../admin/orchestration.md)). + +## Testing and validation + +| Test | Covers | +| --- | --- | +| `functional_tests/test_tabular_merge_engine.py` | Readers, encodings, delimiters, Excel values, sheet selection, legacy XLS conversion, policies, mismatch reporting, provenance, limits, guards, cancellation, spill to disk at 120,000 rows. | +| `functional_tests/test_orchestration_tabular_merge_capability.py` | Registry and gating, plan validation, real executor runs without model calls, search-bound sources, failure messages, revoked access, and CSV/XLSX rendering of the retained result. | +| `functional_tests/test_v2_admin_actions_parity.py`, `functional_tests/test_admin_settings_pane_variable_scope.py` | Admin field paths, bounds and defaults, and the admin card rendering with and without stored Merge settings. | + +## Known limitations + +- Merge appends rows. It does not match rows across files on a key column. +- Phase 1 requires the same columns in every file; Phase 2 reconciles different ones. +- Excel formulas contribute their last calculated values; a workbook saved without them + merges those cells as blanks. +- Workflows cannot run merges until Phase 3. diff --git a/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md b/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md new file mode 100644 index 000000000..80f573c96 --- /dev/null +++ b/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md @@ -0,0 +1,163 @@ +# V2 File Merge — Phase 1: Same-Structure Spreadsheets + +Version: **0.261.218** + +Implemented in version: **0.261.218**, recorded in `application/single_app/config.py`. + +GitHub issue: [#1619](https://github.com/microsoft/simplechat/issues/1619). Part of +[V2 File Merge](V2_FILE_MERGE.md). + +Dependencies: Chat Orchestration (V2), at least one workspace type, `openpyxl` and +`xlrd` (already in `requirements.txt`). + +## Overview + +Phase 1 lets a V2 chat merge two or more CSV or Excel files that share the same columns +into one table and deliver it as a CSV or Excel file. The planner chooses the new +`tabular_merge` Reason capability, the user reviews the plan, and code — not a model — +appends the rows. + +## Technical specifications + +### The capability + +| Field | Value | +| --- | --- | +| ID | `tabular_merge` | +| Label | Merge spreadsheets | +| Role | Reason | +| Gate | `document_action_capabilities.merge.enabled` and at least one workspace type | +| Document limit | Merge chat limit, enforced by the plan validator for explicit IDs and by the adapter for bound source sets | +| Inputs | `document_ids` (two or more, in merge order) **or** `inputs.sources` bound to a `source-set-v1`; `doc_scope`; `schema_policy` (`by_name` default, `exact_order`); optional `sheet`; `include_source_column` (default `true`); `source_column_name` (default `Source File`) | +| Outputs | `records` (`records-v1`), `report` (`structured-v1`) | +| Model | None. `tabular_merge` is not in `STEP_TASKS`, so Auto routing binds no model. | +| Cost class | Medium; at most two merge steps per plan | + +A plan that names a non-tabular document for a merge is rejected with +`source_kind_invalid` before it runs, and the adapter re-checks that every authorized +source is a `.csv`, `.xlsx`, `.xlsm` or `.xls` tabular document. + +### A typical plan + +```json +{ + "steps": [ + { + "step_id": "merge", + "capability_id": "tabular_merge", + "arguments": {"document_ids": ["east-doc", "west-doc", "north-doc"]} + }, + { + "step_id": "save", + "capability_id": "render_file", + "arguments": { + "file_name": "regional_sales.xlsx", + "output_format": "xlsx", + "profile": "exact_tabular_workbook_v1", + "options": {"sheet_name": "Sales"} + }, + "inputs": {"source": {"binding": { + "version": "orchestration-input-binding-v1", + "step_id": "merge", "output_name": "records", "existing_result": null + }, "allow_partial": false}}, + "outputs": [] + } + ] +} +``` + +When the user describes the files instead of selecting them, a `document_search` step +comes first and the merge binds its `sources` output in place of `document_ids`. The +executor expands that source set into document IDs, applies the Merge chat limit, and +refuses the step if any source's scope or revision changed since the search. + +### How files are read + +- **CSV**: the encoding is taken from a byte-order mark (UTF-8, UTF-16, UTF-32), or tried + as strict UTF-8, then Windows-1252, then Latin-1, and recorded in the report. The + delimiter is the most frequent of comma, semicolon, tab or pipe outside quotes on the + first nonblank line. Quoted delimiters and line breaks inside quotes are kept. Values + are copied exactly. +- **XLSX and XLSM**: opened read-only with calculated values and no external links. + Numbers, booleans, dates and times become canonical text (`3.0` → `3`, `TRUE`, ISO 8601 + dates). Macros are never run. +- **XLS**: read with `xlrd`, with dates converted using the workbook's date system and + error cells kept as their Excel error text. +- **Sheets**: a named `sheet` must exist in every workbook (matched exactly, then + ignoring case); otherwise each workbook's first visible sheet is read. + +### How rows are combined + +- The header is the first nonblank row. Trailing blank header cells are dropped; other + blank headers are named `Column N` with a warning. Duplicate names in one file stop + the merge. +- Headers match after Unicode normalization, collapsing whitespace and ignoring case. + The first file decides the output order and spelling. `exact_order` also requires the + same order. +- If any file differs, every remaining header is still read so the report lists every + mismatch, and nothing is merged. +- Completely blank rows are skipped. Short rows are padded with blanks. A row with extra + nonblank values stops the merge. +- Rows are written to a bounded temporary file during the single read, so the exact row + count is known before the retained result is saved. Small merges stay in memory. + +### Failure messages + +Engine errors map to fixed, application-owned messages. User file names and header text +never appear in them. + +| Code | Raised when | +| --- | --- | +| `merge_schema_mismatch` | Files do not share the same columns (or order, with `exact_order`). | +| `merge_columns_invalid` | Duplicate or overly long headers, no header row, or too many columns. | +| `merge_rows_invalid` | A row has more values than headers, or a value exceeds 32,767 characters. | +| `merge_source_unreadable` | A file is damaged, password-protected, or not really CSV or Excel. | +| `merge_sheet_not_found` | The requested sheet is missing from a workbook. | +| `merge_sources_invalid` | Fewer than two files, duplicates, or a non-spreadsheet. | +| `merge_limit_exceeded` | More files or rows than the Merge chat limits, or an oversized file. | + +### Exact-schema exports + +The planner cannot name the columns of a table it has not seen, so the +`render_file` step uses `exact_tabular_records_v1` (CSV) or `exact_tabular_workbook_v1` +(XLSX). Both export every retained column in retained order; XLSX takes an optional +`sheet_name`, defaulting to `Sheet1`. See the +[Generated File Export Framework](GENERATED_FILE_EXPORT_FRAMEWORK.md). + +### Configuration + +| Setting | Default | Notes | +| --- | --- | --- | +| Enable Merge | On | `document_action_capabilities.merge.enabled` | +| Merge: Chat Document Limit | 10 | `merge.chat_max_documents`, 2–300 | +| Merge: Chat Row Limit | 250,000 | `merge.chat_max_rows`, 1,000–1,000,000 | +| Merge spreadsheets capability | Selected | `chat_orchestration_enabled_capabilities`; an empty list allows it | + +Workflow limits are configurable now and take effect with Phase 3. + +## Usage + +1. Open a V2 chat with **Orchestrate** on and select the files. +2. Ask, for example, "Combine these CSVs into one CSV." +3. Review the **Merge spreadsheets** step and the planned file, then run it. + +See [Merge files in chat](../../guides/merge-files.md). + +## Testing and validation + +```powershell +$env:PYTHONPATH = "$PWD\application\single_app;$PWD\functional_tests" +python -m pytest functional_tests/test_tabular_merge_engine.py functional_tests/test_orchestration_tabular_merge_capability.py -q +python -m pytest functional_tests/test_v2_admin_actions_parity.py functional_tests/test_admin_settings_pane_variable_scope.py -q +``` + +The planner goldens (`test_orchestration_workflow_*_off_golden.py`) were regenerated for +this change. Their diff adds only the `tabular_merge` descriptor, the new export profiles +and recipe, and one planner sentence about merging. + +## Known limitations + +- Columns must match; reconciliation arrives in Phase 2. +- Merge appends rows and never joins on a key column. +- Values are text. Excel formulas contribute their last calculated values. +- Large or recurring merges belong in workflows, which arrive in Phase 3. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 4d17e9a75..57208e8ad 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,22 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.218)** + +#### New Features + +* **Merge Spreadsheets In V2 Chat** + * V2 chat orchestration can now combine two or more CSV or Excel files that share the same columns into one table and deliver it as a CSV or Excel file, for example "merge these three regional sales files into one Excel file". It is a new Reason capability, **Merge spreadsheets** (`tabular_merge`), alongside Analyze and Compare; a following **Create a file** step renders the result. + * Code appends the rows, not a model: every row is copied exactly in the order the files are listed, values stay text so codes such as `007` keep their leading zeros, and a **Source File** column records where each row came from. Columns may appear in any order in each file; files whose columns differ are not merged, and nothing is created. + * Files can be selected, or found by a search step whose source set the merge binds. Every file is authorized again and read through the screening-aware byte reader, at the revision the plan approved. CSV encodings and delimiters are detected, Excel values become canonical text (ISO dates, `TRUE`/`FALSE`), and encrypted or damaged workbooks are refused. + * A new **Merge** entry under **Document Action Capabilities** turns the feature on or off (on by default) and sets how many files and merged rows one chat request may merge (10 files, 250,000 rows by default) and, for upcoming workflow merges, the workflow limits (100 files, 1,000,000 rows). + * Merge failures reach users only as fixed, application-owned messages, such as "The selected files don't all have the same columns, so nothing was merged." + * (Ref: #1619, `functions_tabular_merge.py`, `functions_orchestration_merge.py`, `functions_orchestration_adapters.py`, `functions_orchestration_registry.py`, `functions_orchestration_schema.py`, `functions_orchestration_executor.py`, `functions_document_actions.py`, [V2 File Merge](features/V2_FILE_MERGE.md), [Phase 1](features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md), [Merge files in chat](../guides/merge-files.md)) + +* **Exact-Schema CSV And Excel Exports** + * Two new generated-file profiles, `exact_tabular_records_v1` (CSV) and `exact_tabular_workbook_v1` (XLSX, optional sheet name), export every retained column in its retained order. They let a plan deliver a table whose columns are only known when it runs, such as a merge of files a search found, without the planner having to name the columns. + * (Ref: `functions_generated_export_registry.py`, `functions_structured_file_renderers.py`, `functions_generated_office_adapters.py`, [Generated File Export Framework](features/GENERATED_FILE_EXPORT_FRAMEWORK.md)) + ### **(v0.261.217)** #### New Features diff --git a/docs/guides/create-files-with-orchestration.md b/docs/guides/create-files-with-orchestration.md index 6d0d142bf..40462a184 100644 --- a/docs/guides/create-files-with-orchestration.md +++ b/docs/guides/create-files-with-orchestration.md @@ -117,8 +117,8 @@ result can be rendered directly into every format. | Requested format | Appropriate prepared input | | --- | --- | -| CSV | Complete records with an explicit ordered selection of scalar columns. Spreadsheet-like formulas are protected as data. | -| XLSX | Complete typed records with explicit columns and a sheet name. Strings are not treated as executable formulas. | +| CSV | Complete records with an explicit ordered selection of scalar columns, or, for a table whose columns are only known at run time such as a merge, every retained column in order. Spreadsheet-like formulas are protected as data. | +| XLSX | Complete typed records with explicit columns and a sheet name, or every retained column in order with an optional sheet name. Strings are not treated as executable formulas. | | JSON | Complete records or a structured value, including nested objects and arrays. | | YAML or YML | Complete records or a structured value using the declared safe serialization profile. YML is an extension alias. | | XML | Complete records or a structured value using the typed XML representation. This is not automatic generation of an arbitrary requested XML schema. | diff --git a/docs/guides/index.md b/docs/guides/index.md index acb6a0956..06c549cea 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -50,6 +50,7 @@ Use this section when you want momentum first: each page is a practical path thr | [Use web search]({{ '/guides/use-web-search/' | relative_url }}) | Ground an answer in current public web results, and see exactly what leaves SimpleChat when you do. | Users | | [Review pasted URLs]({{ '/guides/review-pasted-urls/' | relative_url }}) | Use URL Access to review links included in the current chat message. | Users | | [Generate images]({{ '/guides/generate-images/' | relative_url }}) | Use the chat Image control to request AI-generated images. | Users | +| [Merge files in chat]({{ '/guides/merge-files/' | relative_url }}) | Combine several CSV or Excel files that share the same columns into one CSV or Excel file. | Users | ## Documents and workspaces diff --git a/docs/guides/merge-files.md b/docs/guides/merge-files.md new file mode 100644 index 000000000..8c0910df6 --- /dev/null +++ b/docs/guides/merge-files.md @@ -0,0 +1,91 @@ +--- +layout: page +title: "Merge files in chat" +description: "Combine several CSV or Excel files that share the same columns into one CSV or Excel file." +section: "Guides" +audience: user +version: "0.261.218" +--- + +## What this does + +Merging takes several spreadsheets with the same columns, such as one sales export per +region or one timesheet per month, and stacks their rows into a single table. The chat +then gives you that table as a downloadable CSV or Excel file. + +Code does the merge, not a model. Every row is copied exactly, in the order you list the +files, so codes such as `007` keep their leading zeros, and nothing is summarized, +rounded or rewritten. The number of rows in the result is checked against the files it +came from. + +## Why you would use this + +Use it when you would otherwise open each file and copy-paste rows into one sheet, or ask +chat to "write a combined CSV" and hope nothing was dropped. Merging is exact and fast, +and it keeps a **Source File** column so you can always see which file each row came from. + +It is not a lookup or join. Merging adds rows; it does not match rows from one file to +another on a key column, such as adding each customer's region from a second file. Ask +for that work separately. + +## Before you start + +- Use the V2 chat with **Orchestrate** on. An administrator must enable Chat + Orchestration and leave **Merge spreadsheets** and **Enable Merge** on; both are on by + default. See [orchestration settings]({{ '/admin/orchestration/' | relative_url }}). +- The files must be CSV (`.csv`) or Excel (`.xlsx`, `.xlsm`, `.xls`) documents you can + open in SimpleChat, in a personal, group or public workspace. +- Every file needs the same column headers. The order of the columns can differ. +- Chat merges up to 10 files and 250,000 rows by default. Your administrator may set + different limits. + +## Merge files + +1. Select the files in the chat's document picker, or name them in your message. +2. Ask for the merge and the file you want back, for example: + + > Merge these three regional sales files into one Excel file. + + > Combine the selected CSVs into one CSV. Don't add a column for the file name. + + > Stack the "Data" sheet from each of these workbooks into one CSV. + +3. Review the plan. It shows a **Merge spreadsheets** task followed by the file to + create. Check the files listed, then run the plan. +4. Download the file from the answer. + +If you don't select the files, you can describe them, for example "merge the monthly +timesheet files from my workspace". The plan then searches for matching files first and +merges the ones it finds, so check the plan's search wording before you run it. + +## What you get + +- **Columns** follow the first file's column order and spelling. Headers in the other + files are matched by name, ignoring capitalization and extra spaces. +- **Source File** is added as the first column unless you ask not to include it. If your + files already have a column with that name, the new column is called + **Source File (2)** instead. +- **Rows** appear file by file, in the order the files were listed. Completely blank + rows are skipped, and rows with fewer values than columns are padded with blanks. +- **Excel values** are copied as plain text. Dates become `YYYY-MM-DD`, true and false + become `TRUE` and `FALSE`, and formulas contribute their last calculated value. +- **Workbooks** contribute their first visible sheet unless you name a sheet, and the + same sheet name is used in every workbook. + +## If the merge doesn't run + +| Message | What to do | +| --- | --- | +| The selected files don't all have the same columns | Check that every file has the same headers. Remove the files that differ, or fix their headers, and ask again. | +| A selected file couldn't be read as a CSV or Excel file | The file may be damaged or password-protected. Open and save it again without a password. | +| A selected workbook doesn't have the requested sheet | Check the sheet name, or leave it out to use each workbook's first visible sheet. | +| The merge is larger than chat allows | Merge fewer or smaller files at a time, or ask your administrator about the Merge limits. | +| Merging needs at least two different CSV or Excel files | Select at least two spreadsheets. Word, PDF and other documents can't be merged as tables. | + +Nothing is created when a merge stops, so there is never a partial file to clean up. + +## Related + +- [Create files with orchestration]({{ '/guides/create-files-with-orchestration/' | relative_url }}) +- [Orchestration settings]({{ '/admin/orchestration/' | relative_url }}) +- [Document Action Capabilities]({{ '/admin/agents-actions/' | relative_url }}#document-action-capabilities-card) diff --git a/functional_tests/test_admin_settings_pane_variable_scope.py b/functional_tests/test_admin_settings_pane_variable_scope.py index 5aca4fe27..f64ce0308 100644 --- a/functional_tests/test_admin_settings_pane_variable_scope.py +++ b/functional_tests/test_admin_settings_pane_variable_scope.py @@ -191,7 +191,7 @@ def test_document_action_capabilities_resolve_in_the_actions_pane(): actions_markup = (PANES_DIR / "actions.html").read_text(encoding="utf-8") agents_markup = (PANES_DIR / "agents.html").read_text(encoding="utf-8") - for capability_name in ("analyze_capability", "comparison_capability"): + for capability_name in ("analyze_capability", "comparison_capability", "merge_capability"): assert re.search(rf"\{{%-?\s*set\s+{capability_name}\s*=", actions_markup), ( f"actions.html uses {capability_name} but never sets it. It has to be " "set in this pane, because a sibling pane's {% set %} is not visible here." @@ -233,6 +233,13 @@ def test_capability_panes_render_as_the_parent_composes_them(): "chat_max_documents": 8, "workflow_max_documents": 40, }, + "merge": { + "enabled": True, + "chat_max_documents": 12, + "workflow_max_documents": 140, + "chat_max_rows": 90000, + "workflow_max_rows": 900000, + }, }, "agents_page_promoted_popular_agents": [], } @@ -255,11 +262,33 @@ def test_capability_panes_render_as_the_parent_composes_them(): assert 'id="document-action-capabilities-card"' in markup, ( "The Document Action Capabilities card did not render." ) - for expected_value in ('value="25"', 'value="120"', 'value="8"', 'value="40"'): + for expected_value in ( + 'value="25"', 'value="120"', 'value="8"', 'value="40"', + 'value="12"', 'value="140"', 'value="90000"', 'value="900000"', + ): assert expected_value in markup, ( f"Configured capability limit {expected_value} did not reach the rendered markup." ) + # Stored settings written before Merge existed have no "merge" entry. The page must + # still render, showing Merge's defaults rather than failing the whole Admin Settings. + legacy = { + **settings, + "document_action_capabilities": { + key: value for key, value in settings["document_action_capabilities"].items() if key != "merge" + }, + } + legacy_markup = parent.render( + settings=legacy, app_settings=legacy, admin_landing_tab="secrets", user_settings={}, mcp_ui_enabled=False, + ) + for expected_value in ('value="10"', 'value="100"', 'value="250000"', 'value="1000000"'): + assert expected_value in legacy_markup, ( + f"The Merge default {expected_value} did not render for settings saved before Merge." + ) + assert re.search(r'id="document_action_merge_enabled"[^>]*checked', legacy_markup), ( + "Merge must show as enabled by default when the stored settings predate it." + ) + print("Both panes render together with their capability limits intact.") diff --git a/functional_tests/test_orchestration_tabular_merge_capability.py b/functional_tests/test_orchestration_tabular_merge_capability.py new file mode 100644 index 000000000..e62333b43 --- /dev/null +++ b/functional_tests/test_orchestration_tabular_merge_capability.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +# test_orchestration_tabular_merge_capability.py +""" +Functional test for the tabular_merge chat orchestration capability. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that tabular_merge is a gated Reason capability alongside Analyze +and Compare: plans name its sources explicitly or bind a search's source set, only +tabular sources are admitted, the administrator's Merge limits apply, the real +executor retains the merged rows and a merge report without any model call, merge +failures reach users only as application-owned messages, and the retained rows +render to CSV and XLSX through the exact-schema export profiles. +""" + +import csv +import io +import sys +from types import ModuleType + +import pytest + +from test_orchestration_dependency_runtime import binding, execute, runtime # noqa: F401 +from test_support.versioning import assert_app_version_at_least + + +AVAILABLE = ['tabular_merge', 'document_search', 'compose', 'render_file'] +FILES = { + 'east-doc': ('east.csv', b'Region,Code,Amount\r\nEast,007,10.50\r\nEast,008,"1,200"\r\n'), + 'west-doc': ('west.csv', b'Amount;Region;Code\n5;West;0099\n'), + 'notes-doc': ('notes.docx', b'not a spreadsheet'), + 'other-doc': ('other.csv', b'Region,Total\nNorth,1\n'), +} + + +def merge_step(document_ids=None, **arguments): + step = {'step_id': 'merge', 'capability_id': 'tabular_merge', 'arguments': dict(arguments)} + if document_ids is not None: + step['arguments']['document_ids'] = list(document_ids) + return step + + +def tabular_case(runtime, steps, *, settings=None, available=AVAILABLE): + case = runtime.make(steps, settings=settings, available=available) + for document_id, (file_name, _) in FILES.items(): + case.fixture.sources[document_id] = { + 'document_id': document_id, 'scope': 'personal', 'scope_id': 'owner', + 'source_version': 1, 'source_revision': f'{document_id}-revision', + 'file_name': file_name, + 'source_kind': 'tabular' if file_name.endswith('.csv') else 'narrative', + } + reads = [] + + def reader(source, user_id, group_id=None, public_workspace_id=None, *, purpose): + assert user_id == 'owner' and purpose == 'native' + reads.append(source['document_id']) + return {'id': source['document_id']}, FILES[source['document_id']][1] + + case.context.merge_source_reader = reader + case.reads = reads + return case + + +def test_version_includes_tabular_merge(): + assert_app_version_at_least('0.261.218') + + +def test_tabular_merge_is_a_gated_reason_capability(runtime): + registry = runtime.registry + capability = registry.get_capability('tabular_merge') + assert capability['role'] == registry.ROLE_REASON + assert capability['label'] == 'Merge spreadsheets' + assert capability['document_action_type'] == 'merge' + assert capability['result_outputs'] == {'records': 'records-v1', 'report': 'structured-v1'} + assert capability['result_input_kinds'] == {'sources': ('source-set-v1',)} + assert capability['inputs']['properties']['document_ids']['minItems'] == 2 + assert capability['adapter'] == 'tabular_merge' + assert 'tabular_merge' in runtime.contracts.REASON_CAPABILITIES + + settings = {'enable_chat_orchestration': True, 'enable_user_workspace': True} + available = {item['id'] for item in registry.resolve_available_capabilities(settings)} + assert 'tabular_merge' in available + disabled = {**settings, 'document_action_capabilities': {'merge': {'enabled': False}}} + assert 'tabular_merge' not in {item['id'] for item in registry.resolve_available_capabilities(disabled)} + no_workspaces = {'enable_chat_orchestration': True} + assert 'tabular_merge' not in {item['id'] for item in registry.resolve_available_capabilities(no_workspaces)} + + projection = registry.build_capability_client_projection([capability]) + assert projection == [{ + 'id': 'tabular_merge', 'label': 'Merge spreadsheets', 'role': 'reason', + 'summary': capability['summary'], 'cost': capability['cost_class'], + }] + assert registry.get_capability_document_limit(capability, settings=settings) == 10 + narrowed = {**settings, 'document_action_capabilities': {'merge': {'chat_max_documents': 4}}} + assert registry.get_capability_document_limit(capability, settings=narrowed) == 4 + + +def test_plans_name_sources_explicitly_or_bind_a_source_set(runtime): + schema = runtime.schema + plan = runtime.make([merge_step(['east-doc', 'west-doc'])], available=AVAILABLE).plan + arguments = plan['steps'][0]['arguments'] + assert arguments == { + 'document_ids': ['east-doc', 'west-doc'], 'doc_scope': 'all', 'schema_policy': 'by_name', + 'include_source_column': True, 'source_column_name': 'Source File', + } + assert [output['name'] for output in plan['steps'][0]['outputs']] == ['records', 'report'] + + with pytest.raises(schema.PlanValidationError) as missing: + runtime.make([merge_step()], available=AVAILABLE) + assert missing.value.code == 'source_binding_required' + + search = {'step_id': 'search', 'capability_id': 'document_search', 'arguments': {'query': 'Q3 sales files'}} + bound = merge_step() + bound['inputs'] = {'sources': {'binding': binding('search', 'sources'), 'allow_partial': False}} + plan = runtime.make([search, bound], available=AVAILABLE).plan + assert plan['steps'][1]['depends_on'] == ['search'] + + both = merge_step(['east-doc', 'west-doc']) + both['inputs'] = bound['inputs'] + with pytest.raises(schema.PlanValidationError): + runtime.make([search, both], available=AVAILABLE) + with pytest.raises(schema.PlanValidationError): + runtime.make([merge_step(['east-doc'])], available=AVAILABLE) + with pytest.raises(schema.PlanValidationError): + runtime.make([merge_step(['east-doc', 'west-doc'], schema_policy='union')], available=AVAILABLE) + + +def test_the_chat_document_limit_and_source_kinds_are_enforced_at_planning(runtime): + schema = runtime.schema + ids = [f'file-{index}' for index in range(4)] + settings = {'document_action_capabilities': {'merge': {'chat_max_documents': 3}}} + with pytest.raises(schema.PlanValidationError) as limited: + runtime.make([merge_step(ids)], settings=settings, available=AVAILABLE) + assert 'document limit' in str(limited.value) + + plan = runtime.make([merge_step(['east-doc', 'notes-doc'])], available=AVAILABLE).plan + with pytest.raises(schema.PlanValidationError) as kinds: + schema.validate_plan_document_source_kinds(plan, {'east-doc': 'tabular', 'notes-doc': 'narrative'}) + assert kinds.value.code == 'source_kind_invalid' + schema.validate_plan_document_source_kinds(plan, {'east-doc': 'tabular'}) + schema.validate_plan_document_source_kinds(plan, {'east-doc': 'tabular', 'notes-doc': 'tabular'}) + + +def test_merge_then_exact_render_plans_validate_without_naming_columns(runtime): + render = { + 'step_id': 'save', 'capability_id': 'render_file', + 'arguments': {'file_name': 'merged.xlsx', 'output_format': 'xlsx', 'profile': 'exact_tabular_workbook_v1', + 'options': {}}, + 'inputs': {'source': {'binding': binding('merge', 'records'), 'allow_partial': False}}, + 'outputs': [], + } + plan = runtime.make([merge_step(['east-doc', 'west-doc']), render], available=AVAILABLE).plan + assert plan['steps'][1]['depends_on'] == ['merge'] + csv_render = {**render, 'arguments': { + 'file_name': 'merged.csv', 'output_format': 'csv', 'profile': 'exact_tabular_records_v1', 'options': {}, + }} + runtime.make([merge_step(['east-doc', 'west-doc']), csv_render], available=AVAILABLE) + with pytest.raises(runtime.schema.PlanValidationError): + runtime.make([merge_step(['east-doc', 'west-doc']), {**render, 'arguments': { + 'file_name': 'merged.csv', 'output_format': 'csv', 'profile': 'tabular_records_v1', 'options': {}, + }}], available=AVAILABLE) + + +def test_executor_retains_exact_rows_and_a_report_without_a_model(runtime): + case = tabular_case(runtime, [merge_step(['east-doc', 'west-doc'])]) + result = execute(runtime, case) + assert result['status'] == 'completed', result + assert case.model.calls == [] + assert case.reads == ['east-doc', 'west-doc'] + step = result['steps'][0] if isinstance(result.get('steps'), list) else None + if step is not None: + assert 'Merged 3 row(s) from 2 file(s)' in step.get('summary', '') + + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + assert task.status == 'complete' + service = case.fixture.restart() + rows = list(service.open_result(task.output('records')).iter_records()) + assert rows == [ + {'Source File': 'east.csv', 'Region': 'East', 'Code': '007', 'Amount': '10.50'}, + {'Source File': 'east.csv', 'Region': 'East', 'Code': '008', 'Amount': '1,200'}, + {'Source File': 'west.csv', 'Region': 'West', 'Code': '0099', 'Amount': '5'}, + ] + records = task.output('records') + assert records.item_count == 3 + assert [column.name for column in records.columns] == ['Source File', 'Region', 'Code', 'Amount'] + assert records.completeness.coverage.unit == 'sources' + report = service.open_result(task.output('report')).read_value() + assert report['status'] == 'merged' + assert report['totals'] == {'sources': 2, 'rows': 3, 'columns': 4, 'blank_rows_skipped': 0} + assert [entry['file_name'] for entry in report['sources']] == ['east.csv', 'west.csv'] + metadata = service.open_result(task.output('records')).metadata() + assert metadata['origin'] == 'grounded' + + +def test_merge_failures_use_application_owned_messages(runtime): + case = tabular_case(runtime, [merge_step(['east-doc', 'other-doc'])]) + result = execute(runtime, case) + assert result['status'] == 'failed' + assert result['failure']['code'] == 'merge_schema_mismatch' + assert 'other.csv' not in result['message'] and 'Total' not in result['message'] + assert "don't all have the same columns" in result['message'] + + case = tabular_case(runtime, [merge_step(['east-doc', 'notes-doc'])]) + result = execute(runtime, case) + assert result['status'] == 'failed' + assert result['failure']['code'] == 'merge_sources_invalid' + assert case.reads == [] + + +def test_merge_reads_are_refused_when_a_source_is_unavailable(runtime): + from functions_orchestration_results import ResultUnavailableError + + case = tabular_case(runtime, [merge_step(['east-doc', 'west-doc'])]) + case.fixture.denied.add('west-doc') + # The executor re-authorizes every planned source before any step runs. + with pytest.raises(ResultUnavailableError): + execute(runtime, case) + assert case.reads == [] + + +def test_merge_can_bind_the_sources_a_search_found(runtime, monkeypatch): + search = {'step_id': 'search', 'capability_id': 'document_search', 'arguments': {'query': 'regional sales'}} + bound = merge_step() + bound['inputs'] = {'sources': {'binding': binding('search', 'sources'), 'allow_partial': False}} + case = tabular_case(runtime, [search, bound]) + module = ModuleType('functions_search') + module.hybrid_search = lambda *args, **kwargs: [ + {'document_id': 'east-doc', 'id': 'hit-1', 'chunk_text': 'East sales'}, + {'document_id': 'west-doc', 'id': 'hit-2', 'chunk_text': 'West sales'}, + ] + monkeypatch.setitem(sys.modules, 'functions_search', module) + result = execute(runtime, case) + assert result['status'] == 'completed', result + assert case.reads == ['east-doc', 'west-doc'] + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + assert task.output('records').item_count == 3 + + +def test_bound_sources_still_respect_the_chat_file_limit(runtime, monkeypatch): + search = {'step_id': 'search', 'capability_id': 'document_search', 'arguments': {'query': 'regional sales'}} + bound = merge_step() + bound['inputs'] = {'sources': {'binding': binding('search', 'sources'), 'allow_partial': False}} + settings = {'document_action_capabilities': {'merge': {'chat_max_documents': 2}}} + case = tabular_case(runtime, [search, bound], settings=settings) + module = ModuleType('functions_search') + module.hybrid_search = lambda *args, **kwargs: [ + {'document_id': document_id, 'id': f'hit-{document_id}', 'chunk_text': 'sales'} + for document_id in ('east-doc', 'west-doc', 'other-doc') + ] + monkeypatch.setitem(sys.modules, 'functions_search', module) + result = execute(runtime, case) + assert result['status'] == 'failed' + assert result['failure']['code'] == 'merge_limit_exceeded' + assert case.reads == [] + + +@pytest.mark.parametrize('output_format, profile', [ + ('csv', 'exact_tabular_records_v1'), ('xlsx', 'exact_tabular_workbook_v1'), +]) +def test_retained_merge_renders_every_column_in_order(runtime, output_format, profile): + from functions_generated_file_exports import GeneratedFileExportRequest, build_generated_file_export + from functions_orchestration_export_sources import open_orchestration_export_source + + case = tabular_case(runtime, [merge_step(['east-doc', 'west-doc'])]) + result = execute(runtime, case) + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + source = open_orchestration_export_source(case.fixture.restart(), task.output('records')) + with build_generated_file_export( + source=source, export_request=GeneratedFileExportRequest(output_format, profile), + max_output_bytes=8 * 1024 * 1024, + ) as output: + content = output.file_content.read() + source.require_complete_consumption() + assert output.record_count == 3 + if output_format == 'csv': + rows = list(csv.reader(io.StringIO(content.decode('utf-8')))) + assert rows == [ + ['Source File', 'Region', 'Code', 'Amount'], + ['east.csv', 'East', '007', '10.50'], + ['east.csv', 'East', '008', '1,200'], + ['west.csv', 'West', '0099', '5'], + ] + else: + from openpyxl import load_workbook + + workbook = load_workbook(io.BytesIO(content), read_only=True) + sheet = workbook['Sheet1'] + values = [list(row) for row in sheet.iter_rows(values_only=True)] + workbook.close() + assert values[0] == ['Source File', 'Region', 'Code', 'Amount'] + assert values[1] == ['east.csv', 'East', '007', '10.50'] + assert values[3] == ['west.csv', 'West', '0099', '5'] + + +if __name__ == '__main__': + sys.exit(pytest.main([__file__, '-q'])) diff --git a/functional_tests/test_support/orchestration_workflow_results_off_golden.json b/functional_tests/test_support/orchestration_workflow_results_off_golden.json index babd13bbc..b3a749dc9 100644 --- a/functional_tests/test_support/orchestration_workflow_results_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_results_off_golden.json @@ -1366,7 +1366,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -1675,7 +1678,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -1940,7 +1946,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -2574,6 +2583,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -3235,6 +3321,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -3273,7 +3360,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -3345,7 +3433,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -4194,6 +4283,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -4800,6 +4966,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -4838,7 +5005,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -4910,7 +5078,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -5695,6 +5864,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -6265,6 +6511,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -6303,7 +6550,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -6375,7 +6623,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -6990,6 +7239,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -7596,6 +7922,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -7633,7 +7960,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -7705,7 +8033,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -8198,6 +8527,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -8218,6 +8548,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -8238,6 +8569,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -8258,6 +8590,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -8278,6 +8611,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -8322,6 +8656,13 @@ "summary": "Compute over CSV or Excel data rather than reading it as prose.", "cost": "medium" }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "cost": "medium" + }, { "id": "web_search", "label": "Search the web", @@ -8401,6 +8742,13 @@ "summary": "Compute over CSV or Excel data rather than reading it as prose.", "cost": "medium" }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "cost": "medium" + }, { "id": "web_search", "label": "Search the web", diff --git a/functional_tests/test_support/orchestration_workflow_runs_off_golden.json b/functional_tests/test_support/orchestration_workflow_runs_off_golden.json index e21744fb5..917155ed6 100644 --- a/functional_tests/test_support/orchestration_workflow_runs_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_runs_off_golden.json @@ -665,7 +665,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -951,7 +954,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -1585,6 +1591,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -2210,6 +2293,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -2247,7 +2331,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -2319,7 +2404,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -3151,6 +3237,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -3721,6 +3884,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -3758,7 +3922,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -3830,7 +3995,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -4049,6 +4215,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -4068,6 +4235,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -4087,6 +4255,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -4106,6 +4275,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", diff --git a/functional_tests/test_support/orchestration_workflow_setting_off_golden.json b/functional_tests/test_support/orchestration_workflow_setting_off_golden.json index 9b9bb7818..192296c7b 100644 --- a/functional_tests/test_support/orchestration_workflow_setting_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_setting_off_golden.json @@ -82,7 +82,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each", + "do not admit native tabular inputs. To combine the rows of several tabular sources that share", + "the same columns into one table or file, use tabular_merge with every one of those sources, never", + "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", "identifiers. A preparation step sees only its own sources, so never ask it to compare them", @@ -712,6 +715,83 @@ } ] }, + { + "id": "tabular_merge", + "label": "Merge spreadsheets", + "role": "reason", + "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 2, + "uniqueItems": true, + "description": "The CSV or Excel files to merge, in the order their rows should appear." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "schema_policy": { + "type": "string", + "enum": [ + "by_name", + "exact_order" + ], + "default": "by_name", + "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "include_source_column": { + "type": "boolean", + "default": true, + "description": "Add a column holding each row's file name." + }, + "source_column_name": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "default": "Source File" + } + }, + "additionalProperties": false + }, + "cost": "medium", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-merge-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "records": "records-v1", + "report": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "web_search", "label": "Search the web", @@ -1282,6 +1362,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", @@ -1318,7 +1399,8 @@ "records-v1" ], "profiles": [ - "tabular_records_v1" + "tabular_records_v1", + "exact_tabular_records_v1" ], "embeds_images": false, "status": "unavailable", @@ -1390,7 +1472,8 @@ "records-v1" ], "profiles": [ - "tabular_workbook_v1" + "tabular_workbook_v1", + "exact_tabular_workbook_v1" ], "embeds_images": false, "status": "unavailable", @@ -1530,6 +1613,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_merge", "web_search", "action_invoke", "agent_invoke", diff --git a/functional_tests/test_tabular_merge_engine.py b/functional_tests/test_tabular_merge_engine.py new file mode 100644 index 000000000..a96581c35 --- /dev/null +++ b/functional_tests/test_tabular_merge_engine.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +# test_tabular_merge_engine.py +""" +Functional test for the deterministic tabular merge engine. +Version: 0.261.218 +Implemented in: 0.261.218 + +This test ensures that functions_tabular_merge appends the rows of same-structure +CSV and Excel files exactly: values stay text (leading zeros, codes and Unicode +survive), columns match by name or exact order, every mismatching file is reported +before anything is merged, encodings and delimiters are detected, workbook sheets are +selected deterministically, and every limit, guard and cancellation fails closed. +""" + +import codecs +import io +import os +import sys +from datetime import date, datetime + +import pytest + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "application", "single_app")) + +from test_support.versioning import assert_app_version_at_least + +import functions_tabular_merge as merge +from functions_tabular_merge import ( + TabularMergeCancelled, + TabularMergeError, + TabularMergeLimits, + TabularMergeOptions, + TabularMergeSource, + merge_tabular_sources, +) + + +def csv_source(name, text, encoding="utf-8", source_id=None): + data = text.encode(encoding) if isinstance(text, str) else text + return TabularMergeSource(source_id or name, name, lambda data=data: data) + + +def bytes_source(name, data, source_id=None): + return TabularMergeSource(source_id or name, name, lambda data=data: data) + + +def xlsx_bytes(sheets): + from openpyxl import Workbook + + workbook = Workbook() + first = True + for title, rows, state in sheets: + worksheet = workbook.active if first else workbook.create_sheet() + first = False + worksheet.title = title + worksheet.sheet_state = state + for row in rows: + worksheet.append(row) + buffer = io.BytesIO() + workbook.save(buffer) + return buffer.getvalue() + + +def records(result): + return list(result.iter_records()) + + +def test_version_includes_the_merge_engine(): + assert_app_version_at_least("0.261.218") + + +def test_appends_rows_in_file_order_with_a_source_column(): + first = csv_source("east.csv", "Region,Code,Amount\r\nEast,007,10.50\r\nEast,008,\"1,200\"\r\n") + second = csv_source("west.csv", "Region,Code,Amount\nWest,0099,5\n") + with merge_tabular_sources([first, second]) as result: + assert result.columns == ("Source File", "Region", "Code", "Amount") + assert result.row_count == 3 + assert records(result) == [ + {"Source File": "east.csv", "Region": "East", "Code": "007", "Amount": "10.50"}, + {"Source File": "east.csv", "Region": "East", "Code": "008", "Amount": "1,200"}, + {"Source File": "west.csv", "Region": "West", "Code": "0099", "Amount": "5"}, + ] + report = result.report + assert report["version"] == "tabular-merge-report-v1" + assert report["status"] == "merged" + assert report["totals"] == {"sources": 2, "rows": 3, "columns": 4, "blank_rows_skipped": 0} + assert [entry["rows"] for entry in report["sources"]] == [2, 1] + assert report["limitations"] == [] + + +def test_matches_columns_by_name_in_any_order_and_reports_reordering(): + first = csv_source("a.csv", "Region,Code\nNorth,1\n") + second = csv_source("b.csv", " code ;REGION\n2;South\n") + with merge_tabular_sources([first, second], options=TabularMergeOptions(include_source_column=False)) as result: + assert result.columns == ("Region", "Code") + assert records(result) == [{"Region": "North", "Code": "1"}, {"Region": "South", "Code": "2"}] + assert [entry["order_differs"] for entry in result.report["sources"]] == [False, True] + assert result.report["sources"][1]["delimiter"] == ";" + + +def test_exact_order_rejects_reordered_columns(): + first = csv_source("a.csv", "Region,Code\nNorth,1\n") + second = csv_source("b.csv", "Code,Region\n2,South\n") + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([first, second], options=TabularMergeOptions(schema_policy="exact_order")) + assert caught.value.code == "schema_mismatch" + assert caught.value.report["sources"][1]["order_differs"] is True + assert "different order" in caught.value.message + + +def test_schema_mismatch_reports_every_file_and_merges_nothing(): + reference = csv_source("ref.csv", "Region,Code,Amount\nEast,1,2\n") + extra = csv_source("extra.csv", "Region,Code,Total,Notes\nX,1,2,3\n") + matching = csv_source("ok.csv", "Amount,Code,Region\n3,4,West\n") + missing = csv_source("missing.csv", "Region,Amount\nY,1\n") + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([reference, extra, matching, missing]) + error = caught.value + assert error.code == "schema_mismatch" + statuses = [(entry["file_name"], entry["status"]) for entry in error.report["sources"]] + assert statuses == [ + ("ref.csv", "compatible"), ("extra.csv", "schema_mismatch"), + ("ok.csv", "compatible"), ("missing.csv", "schema_mismatch"), + ] + assert error.report["sources"][1]["missing_columns"] == ["Amount"] + assert error.report["sources"][1]["extra_columns"] == ["Total", "Notes"] + assert error.report["sources"][3]["missing_columns"] == ["Code"] + assert all(entry["rows"] == 0 for entry in error.report["sources"]) + assert error.report["status"] == "failed" + assert "Nothing was merged" in error.message + + +@pytest.mark.parametrize( + ("payload", "encoding_name"), + [ + ("Name,City\nZoë,Köln\n".encode("utf-8"), "utf-8"), + (codecs.BOM_UTF8 + "Name,City\nZoë,Köln\n".encode("utf-8"), "utf-8-sig"), + ("Name,City\nZoë,Köln\n".encode("utf-16"), "utf-16"), + ("Name,City\nZoë,Köln\n".encode("cp1252"), "cp1252"), + ], +) +def test_detects_text_encodings(payload, encoding_name): + other = csv_source("other.csv", "Name,City\nAnn,Oslo\n") + with merge_tabular_sources([bytes_source("enc.csv", payload), other]) as result: + assert records(result)[0] == {"Source File": "enc.csv", "Name": "Zoë", "City": "Köln"} + assert result.report["sources"][0]["encoding"] == encoding_name + + +@pytest.mark.parametrize("delimiter", [",", ";", "\t", "|"]) +def test_detects_delimiters_and_keeps_quoted_delimiters(delimiter): + text = f'Name{delimiter}Note\n"Smith{delimiter} Jr"{delimiter}"line one\nline two"\n' + other = csv_source("b.csv", "Name,Note\nAnn,x\n") + with merge_tabular_sources([csv_source("a.csv", text), other]) as result: + first = records(result)[0] + assert first["Name"] == f"Smith{delimiter} Jr" + assert first["Note"] == "line one\nline two" + assert result.report["sources"][0]["delimiter"] == delimiter + + +def test_rejects_binary_data_mislabeled_as_csv(): + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([bytes_source("bad.csv", b"PK\x03\x04\x00\x00junk"), csv_source("b.csv", "A\n1\n")]) + assert caught.value.code == "unreadable_csv" + + +def test_reads_xlsx_values_as_canonical_text(): + workbook = xlsx_bytes([( + "Data", + [ + ["Code", "Amount", "When", "Flag", "Ratio", "Formula"], + ["007", 3.0, datetime(2024, 1, 31), True, 0.1, "=1+1"], + ["008", 12, datetime(2024, 2, 1, 9, 30, 15), False, 1e-07, None], + [None, None, None, None, None, None], + ["009", -2.5, date(2023, 12, 25), None, 123456789012.0, None], + ], + "visible", + )]) + other = csv_source("b.csv", "Code,Amount,When,Flag,Ratio,Formula\n010,1,2024-03-01,TRUE,1,\n") + with merge_tabular_sources([bytes_source("book.xlsx", workbook), other]) as result: + rows = records(result) + assert rows[0] == { + "Source File": "book.xlsx", "Code": "007", "Amount": "3", "When": "2024-01-31", + "Flag": "TRUE", "Ratio": "0.1", "Formula": "", + } + assert rows[1]["When"] == "2024-02-01T09:30:15" + assert rows[1]["Amount"] == "12" + assert rows[1]["Flag"] == "FALSE" + assert rows[1]["Ratio"] == "1e-07" + assert rows[2]["When"] == "2023-12-25" + assert rows[2]["Amount"] == "-2.5" + assert rows[2]["Ratio"] == "123456789012" + report = result.report + assert report["sources"][0]["sheet"] == "Data" + assert report["sources"][0]["blank_rows_skipped"] == 1 + assert report["limitations"], "Workbook merges must disclose how formulas and types were read." + + +def test_selects_the_first_visible_sheet_or_a_named_sheet(): + workbook = xlsx_bytes([ + ("Hidden", [["A"], ["hidden"]], "hidden"), + ("Visible", [["A"], ["visible"]], "visible"), + ("Second", [["A"], ["second"]], "visible"), + ]) + other = csv_source("b.csv", "A\ncsv\n") + with merge_tabular_sources([bytes_source("w.xlsx", workbook), other]) as result: + assert records(result)[0]["A"] == "visible" + options = TabularMergeOptions(sheet="second") + with merge_tabular_sources([bytes_source("w.xlsx", workbook), other], options=options) as result: + assert records(result)[0]["A"] == "second" + assert result.report["sources"][0]["sheet"] == "Second" + + +def test_missing_named_sheet_lists_the_available_sheets(): + workbook = xlsx_bytes([("Data", [["A"], ["1"]], "visible")]) + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources( + [bytes_source("w.xlsx", workbook), csv_source("b.csv", "A\n2\n")], + options=TabularMergeOptions(sheet="Totals"), + ) + assert caught.value.code == "sheet_not_found" + assert caught.value.details["available_sheets"] == ["Data"] + + +def test_xls_reader_converts_legacy_cell_types(monkeypatch): + import xlrd + + class Cell: + def __init__(self, ctype, value): + self.ctype = ctype + self.value = value + + rows = [ + [Cell(xlrd.XL_CELL_TEXT, "Code"), Cell(xlrd.XL_CELL_TEXT, "Amount"), Cell(xlrd.XL_CELL_TEXT, "When"), + Cell(xlrd.XL_CELL_TEXT, "Flag"), Cell(xlrd.XL_CELL_TEXT, "Error")], + [Cell(xlrd.XL_CELL_TEXT, "007"), Cell(xlrd.XL_CELL_NUMBER, 4.0), Cell(xlrd.XL_CELL_DATE, 45322.0), + Cell(xlrd.XL_CELL_BOOLEAN, 1), Cell(xlrd.XL_CELL_ERROR, 0x2A)], + [Cell(xlrd.XL_CELL_EMPTY, ""), Cell(xlrd.XL_CELL_BLANK, ""), Cell(xlrd.XL_CELL_EMPTY, ""), + Cell(xlrd.XL_CELL_EMPTY, ""), Cell(xlrd.XL_CELL_EMPTY, "")], + ] + + class Sheet: + visibility = 0 + nrows = len(rows) + + def row(self, index): + return rows[index] + + class Book: + datemode = 0 + + def sheet_names(self): + return ["Legacy"] + + def sheet_by_index(self, index): + return Sheet() + + def sheet_by_name(self, name): + assert name == "Legacy" + return Sheet() + + def release_resources(self): + return None + + monkeypatch.setattr(xlrd, "open_workbook", lambda **kwargs: Book()) + legacy = bytes_source("old.xls", merge._OLE_MAGIC + b"\x00" * 64) + other = csv_source("b.csv", "Code,Amount,When,Flag,Error\n1,2,3,4,5\n") + with merge_tabular_sources([legacy, other]) as result: + first = records(result)[0] + assert first == { + "Source File": "old.xls", "Code": "007", "Amount": "4", "When": "2024-01-31", + "Flag": "TRUE", "Error": "#N/A", + } + assert result.report["sources"][0]["blank_rows_skipped"] == 1 + + +def test_password_protected_and_damaged_workbooks_fail_clearly(): + other = csv_source("b.csv", "A\n1\n") + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([bytes_source("locked.xlsx", merge._OLE_MAGIC + b"\x00" * 32), other]) + assert caught.value.code == "encrypted_workbook" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([bytes_source("broken.xlsx", b"not a workbook"), other]) + assert caught.value.code == "unreadable_workbook" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([bytes_source("broken.xls", b"not xls"), other]) + assert caught.value.code == "unreadable_workbook" + + +def test_workbook_archive_guard_rejects_oversized_expansion(): + workbook = xlsx_bytes([("Data", [["A"], ["1"]], "visible")]) + limits = TabularMergeLimits(max_workbook_uncompressed_bytes=64) + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([bytes_source("w.xlsx", workbook), csv_source("b.csv", "A\n2\n")], limits=limits) + assert caught.value.code == "workbook_too_large" + + +def test_blank_rows_are_skipped_short_rows_padded_and_extra_values_refused(): + first = csv_source("a.csv", "A,B,C,,\n1,2,3,,\n,,\n4\n") + second = csv_source("b.csv", "A,B,C\n5,6,7\n") + with merge_tabular_sources([first, second], options=TabularMergeOptions(include_source_column=False)) as result: + assert records(result) == [ + {"A": "1", "B": "2", "C": "3"}, + {"A": "4", "B": "", "C": ""}, + {"A": "5", "B": "6", "C": "7"}, + ] + source_entry = result.report["sources"][0] + assert source_entry["blank_rows_skipped"] == 1 + assert source_entry["short_rows_padded"] == 1 + bad = csv_source("bad.csv", "A,B,C\n1,2,3,surprise\n") + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([second, bad]) + assert caught.value.code == "row_has_extra_values" + + +def test_header_rules(): + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("dup.csv", "A,a\n1,2\n"), csv_source("b.csv", "A\n1\n")]) + assert caught.value.code == "duplicate_columns" + with merge_tabular_sources( + [csv_source("blank.csv", "A,,C\n1,2,3\n"), csv_source("b.csv", "A,Column 2,C\n4,5,6\n")], + ) as result: + assert result.columns == ("Source File", "A", "Column 2", "C") + assert result.report["sources"][0]["warnings"][0]["code"] == "blank_header" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("empty.csv", "\n\n"), csv_source("b.csv", "A\n1\n")]) + assert caught.value.code == "empty_source" + long_header = "é" * 129 # 258 UTF-8 bytes: record column names are capped at 256 bytes. + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("long.csv", f"{long_header}\n1\n"), csv_source("b.csv", "A\n1\n")]) + assert caught.value.code == "header_too_long" + + +def test_source_column_is_renamed_on_collision_or_omitted(): + first = csv_source("a.csv", "Source File,Value\nx,1\n") + second = csv_source("b.csv", "Source File,Value\ny,2\n") + with merge_tabular_sources([first, second]) as result: + assert result.columns == ("Source File (2)", "Source File", "Value") + assert result.report["warnings"][0]["code"] == "source_column_renamed" + assert records(result)[0] == {"Source File (2)": "a.csv", "Source File": "x", "Value": "1"} + options = TabularMergeOptions(include_source_column=False) + with merge_tabular_sources([first, second], options=options) as result: + assert result.columns == ("Source File", "Value") + assert result.report["source_column"] is None + + +def test_header_only_files_merge_to_an_empty_complete_table(): + with merge_tabular_sources([csv_source("a.csv", "A,B\n"), csv_source("b.csv", "A,B\r\n")]) as result: + assert result.row_count == 0 + assert records(result) == [] + assert result.report["status"] == "merged" + + +def test_source_validation(): + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("a.csv", "A\n1\n")]) + assert caught.value.code == "too_few_sources" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("a.csv", "A\n1\n", source_id="same"), csv_source("b.csv", "A\n2\n", source_id="same")]) + assert caught.value.code == "duplicate_source" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([csv_source("a.csv", "A\n1\n"), csv_source("notes.docx", "A\n2\n")]) + assert caught.value.code == "unsupported_format" + with pytest.raises(TabularMergeError) as caught: + TabularMergeOptions(schema_policy="union") + assert caught.value.code == "invalid_options" + + +def test_limits_fail_closed(): + sources = [csv_source(f"{index}.csv", "A\n1\n") for index in range(3)] + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources(sources, limits=TabularMergeLimits(max_sources=2)) + assert caught.value.code == "too_many_sources" + many_rows = csv_source("big.csv", "A\n" + "\n".join(str(value) for value in range(30)) + "\n") + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([many_rows, sources[0]], limits=TabularMergeLimits(max_total_rows=10)) + assert caught.value.code == "row_limit_exceeded" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([many_rows, sources[0]], limits=TabularMergeLimits(max_source_bytes=8)) + assert caught.value.code == "source_too_large" + wide = ",".join(f"C{index}" for index in range(5)) + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources( + [csv_source("w.csv", f"{wide}\n"), csv_source("x.csv", f"{wide}\n")], + limits=TabularMergeLimits(max_columns=5), + ) + assert caught.value.code == "too_many_columns" + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources( + [csv_source("c.csv", "A\n" + "x" * 50 + "\n"), sources[0]], + limits=TabularMergeLimits(max_cell_chars=10), + ) + assert caught.value.code == "cell_too_large" + + +def test_cancellation_stops_between_sources(): + calls = {"count": 0} + + def cancel_after_first(): + calls["count"] += 1 + return calls["count"] > 2 + + sources = [csv_source(f"{index}.csv", "A\n1\n") for index in range(3)] + with pytest.raises(TabularMergeCancelled): + merge_tabular_sources(sources, cancel_requested=cancel_after_first) + + +def test_rows_can_be_read_more_than_once_until_closed_and_progress_is_reported(): + events = [] + result = merge_tabular_sources( + [csv_source("a.csv", "A\n1\n2\n"), csv_source("b.csv", "A\n3\n")], on_progress=events.append, + ) + try: + assert records(result) == records(result) + assert [event["index"] for event in events] == [1, 2] + assert events[-1] == {"index": 2, "total": 2, "file_name": "b.csv", "rows": 1} + finally: + result.close() + with pytest.raises(TabularMergeError) as caught: + list(result.iter_rows()) + assert caught.value.code == "result_closed" + + +def test_large_merges_spill_to_disk_and_keep_exact_counts(): + rows = "\n".join(f"{index:07d},{'x' * 40}" for index in range(120_000)) + first = csv_source("big1.csv", f"Code,Text\n{rows}\n") + second = csv_source("big2.csv", "Code,Text\n0000001,y\n") + with merge_tabular_sources([first, second]) as result: + assert result.row_count == 120_001 + count = 0 + for record in result.iter_records(): + count += 1 + assert count == 120_001 + assert records(result)[0]["Code"] == "0000000" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_v2_admin_actions_parity.py b/functional_tests/test_v2_admin_actions_parity.py index 1488c9abb..34ad01094 100644 --- a/functional_tests/test_v2_admin_actions_parity.py +++ b/functional_tests/test_v2_admin_actions_parity.py @@ -139,6 +139,18 @@ def document_action_default_limits(): } +def merge_default_limits(): + """Return Merge's own document and row defaults and its row bounds.""" + source = DOCUMENT_ACTIONS_MODULE.read_text(encoding="utf-8") + return { + "chat_max_documents": literal_assignment(source, "MERGE_CHAT_DEFAULT_MAX_DOCUMENTS"), + "workflow_max_documents": literal_assignment(source, "MERGE_WORKFLOW_DEFAULT_MAX_DOCUMENTS"), + "chat_max_rows": literal_assignment(source, "MERGE_CHAT_DEFAULT_MAX_ROWS"), + "workflow_max_rows": literal_assignment(source, "MERGE_WORKFLOW_DEFAULT_MAX_ROWS"), + "row_bounds": literal_assignment(source, "MERGE_ROW_LIMIT_BOUNDS"), + } + + def test_removing_a_section_did_not_orphan_its_settings(): """A section can be emptied, but not at the cost of losing a setting. @@ -269,6 +281,27 @@ def test_document_action_fields_write_into_the_nested_container(): "comparison", "workflow_max_documents", ], + "document_action_merge_enabled": ["document_action_capabilities", "merge", "enabled"], + "document_action_merge_chat_max_documents": [ + "document_action_capabilities", + "merge", + "chat_max_documents", + ], + "document_action_merge_workflow_max_documents": [ + "document_action_capabilities", + "merge", + "workflow_max_documents", + ], + "document_action_merge_chat_max_rows": [ + "document_action_capabilities", + "merge", + "chat_max_rows", + ], + "document_action_merge_workflow_max_rows": [ + "document_action_capabilities", + "merge", + "workflow_max_rows", + ], } actual = {field["key"]: field.get("settings_path") for field in fields} @@ -286,6 +319,7 @@ def test_document_action_bounds_match_the_application(): bounds = document_action_bounds() defaults = document_action_default_limits() + merge_defaults = merge_default_limits() schema = fields_module.get_admin_settings_fields() problems = [] @@ -297,18 +331,24 @@ def test_document_action_bounds_match_the_application(): problems.append(f"{field['key']}: default {field.get('default')!r}, expected True") continue - context = "chat" if leaf == "chat_max_documents" else "workflow" - if field.get("min") != bounds[context]["min"]: + if leaf in ("chat_max_rows", "workflow_max_rows"): + expected_bounds = merge_defaults["row_bounds"] + expected_default = merge_defaults[leaf] + else: + context = "chat" if leaf == "chat_max_documents" else "workflow" + expected_bounds = bounds[context] + expected_default = merge_defaults[leaf] if path[1] == "merge" else defaults[context] + if field.get("min") != expected_bounds["min"]: problems.append( - f"{field['key']}: min {field.get('min')!r} != {bounds[context]['min']!r}" + f"{field['key']}: min {field.get('min')!r} != {expected_bounds['min']!r}" ) - if field.get("max") != bounds[context]["max"]: + if field.get("max") != expected_bounds["max"]: problems.append( - f"{field['key']}: max {field.get('max')!r} != {bounds[context]['max']!r}" + f"{field['key']}: max {field.get('max')!r} != {expected_bounds['max']!r}" ) - if field.get("default") != defaults[context]: + if field.get("default") != expected_default: problems.append( - f"{field['key']}: default {field.get('default')!r} != {defaults[context]!r}" + f"{field['key']}: default {field.get('default')!r} != {expected_default!r}" ) assert not problems, ( diff --git a/ui_tests/test_admin_document_action_capabilities_card.py b/ui_tests/test_admin_document_action_capabilities_card.py index 89561fa9f..6e022f794 100644 --- a/ui_tests/test_admin_document_action_capabilities_card.py +++ b/ui_tests/test_admin_document_action_capabilities_card.py @@ -1,13 +1,14 @@ # test_admin_document_action_capabilities_card.py """ UI test for admin document action capabilities placement. -Version: 0.260.019 +Version: 0.261.218 Implemented in: 0.241.089 -Updated in: 0.260.019 +Updated in: 0.261.218 This test ensures the Document Action Capabilities card is visible at the top of the Actions tab, explains that it controls the Action dropdown in Chat and -Workflow, and renders its configured limits. +Workflow, and renders its configured limits, including the Merge file and row +limits added in 0.261.218. The limit assertions are the regression guard for the Admin Settings 500: the card reads values that used to be set in the Agents pane, and sibling @@ -97,6 +98,10 @@ def test_admin_document_action_capabilities_card_is_top_of_actions_tab(playwrigh "document_action_analyze_workflow_max_documents", "document_action_comparison_chat_max_documents", "document_action_comparison_workflow_max_documents", + "document_action_merge_chat_max_documents", + "document_action_merge_workflow_max_documents", + "document_action_merge_chat_max_rows", + "document_action_merge_workflow_max_rows", ) for input_id in limit_input_ids: limit_input = page.locator(f"#{input_id}") @@ -105,8 +110,18 @@ def test_admin_document_action_capabilities_card_is_top_of_actions_tab(playwrigh f"Expected #{input_id} to render a configured limit." ) - for toggle_id in ("document_action_analyze_enabled", "document_action_comparison_enabled"): + for toggle_id in ( + "document_action_analyze_enabled", "document_action_comparison_enabled", "document_action_merge_enabled", + ): expect(page.locator(f"#{toggle_id}")).to_be_visible() + + # The range and number controls stay in step, as they do for Analyze and Comparison. + merge_rows = page.locator("#document_action_merge_chat_max_rows") + merge_rows_range = page.locator("#document_action_merge_chat_max_rows_range") + merge_rows_range.fill("120000") + expect(merge_rows).to_have_value("120000") + expect(page.locator("#document_action_merge_chat_max_rows_value")).to_have_text("120000") + expect(card).to_contain_text("Merge") finally: context.close() browser.close() From 0f6a04a15ff2228835c6881b76207618c678f087 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 20:19:35 -0400 Subject: [PATCH 02/13] Reconcile differently structured spreadsheets in V2 merges (Phase 2, refs #1619) Lets a V2 chat merge CSV and Excel files whose columns differ, and adds a Gather capability that inspects files before merging. - Engine: union and mapped policies, column aliases, header row, every-sheet mode with a Source Sheet column, exclusion of files or sheets that do not fit, duplicate removal (exact rows or key columns, first or last), bounded multi-column sort, nullable columns, and tabular inspection - tabular_inspect (Gather) retains sheets, headers, row counts, samples and a compatibility summary without a model - tabular_column_mapping_v1 compose profile; tabular_merge binds it as a "mapping" input and validates it again before code applies it - Plan validation shares the engine's option rules; a bound mapping must come from a compose step using the profile - Planner recipe and guidance, new catalog failure messages, plan review wording for merge and inspection settings - Fixes the registry capability list test that Phase 1 left stale - Docs, guide, release notes, functional, .mjs and browser UI tests; goldens regenerated and audited Version 0.261.219 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../single_app/admin_settings_fields.py | 10 +- application/single_app/config.py | 2 +- .../functions_orchestration_adapters.py | 151 +- .../functions_orchestration_deliverables.py | 10 + .../functions_orchestration_executor.py | 12 +- .../functions_orchestration_merge.py | 148 +- .../functions_orchestration_planner.py | 7 +- .../functions_orchestration_registry.py | 201 +- .../functions_orchestration_schema.py | 92 +- .../functions_orchestration_services.py | 20 +- .../single_app/functions_tabular_merge.py | 1783 ++++++++++++++--- .../templates/admin/_panes/actions.html | 2 +- .../components/chat/OrchestrationRunView.tsx | 8 +- .../v2_ui/src/lib/orchestrationMerge.ts | 148 ++ docs/admin/agents-actions.md | 14 +- docs/admin/orchestration.md | 33 +- .../features/CHAT_ORCHESTRATION.md | 3 +- docs/explanation/features/V2_FILE_MERGE.md | 49 +- .../V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md | 3 +- .../V2_FILE_MERGE_PHASE_2_RECONCILIATION.md | 230 +++ docs/explanation/release_notes.md | 23 + docs/guides/index.md | 2 +- docs/guides/merge-files.md | 93 +- ...test_orchestration_dependency_allowlist.py | 4 +- ..._orchestration_tabular_merge_capability.py | 9 +- ...st_orchestration_tabular_reconciliation.py | 369 ++++ ...estration_workflow_results_off_golden.json | 1104 +++++++++- ...rchestration_workflow_runs_off_golden.json | 548 ++++- ...estration_workflow_setting_off_golden.json | 273 ++- functional_tests/test_tabular_merge_engine.py | 7 +- .../test_tabular_merge_reconciliation.py | 453 +++++ .../test_v2_orchestration_merge_arguments.mjs | 107 + .../test_v2_orchestration_merge_arguments.py | 275 +++ 33 files changed, 5716 insertions(+), 477 deletions(-) create mode 100644 application/v2_ui/src/lib/orchestrationMerge.ts create mode 100644 docs/explanation/features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md create mode 100644 functional_tests/test_orchestration_tabular_reconciliation.py create mode 100644 functional_tests/test_tabular_merge_reconciliation.py create mode 100644 functional_tests/test_v2_orchestration_merge_arguments.mjs create mode 100644 ui_tests/test_v2_orchestration_merge_arguments.py diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 6fc1246b1..7c3ea6154 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -3827,6 +3827,7 @@ {"value": "document_analyze", "label": "Analyse documents"}, {"value": "document_compare", "label": "Compare documents"}, {"value": "tabular_analyze", "label": "Analyse spreadsheets"}, + {"value": "tabular_inspect", "label": "Inspect spreadsheets"}, {"value": "tabular_merge", "label": "Merge spreadsheets"}, {"value": "web_search", "label": "Search the web"}, {"value": "url_fetch", "label": "Read linked pages"}, @@ -4622,8 +4623,9 @@ "label": "Enable Merge", "help": ( "Lets chat orchestration combine the rows of several CSV or Excel files " - "that share the same columns into one table, then deliver it as a CSV or " - "Excel file. Rows are appended exactly by code, not rewritten by a model." + "into one table, lining up columns that differ when asked, then deliver it " + "as a CSV or Excel file. Rows are appended exactly by code, not rewritten by " + "a model. Also allows inspecting the files' columns first." ), "default": True, "group": "Merge", @@ -4634,8 +4636,8 @@ "type": "number", "label": "Merge: Chat Document Limit", "help": ( - "Most files one chat request may merge. Merging is cheaper than Analyze " - "because no model reads the files, so this can be higher." + "Most files one chat request may merge or inspect. Merging is cheaper than " + "Analyze because no model reads the files, so this can be higher." ), "default": DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_LIMIT, "min": DOCUMENT_ACTION_CHAT_MIN_LIMIT, diff --git a/application/single_app/config.py b/application/single_app/config.py index 4ef56809c..fa1818f47 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -100,7 +100,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.218" +VERSION = "0.261.219" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_orchestration_adapters.py b/application/single_app/functions_orchestration_adapters.py index e1274556f..f6db57281 100644 --- a/application/single_app/functions_orchestration_adapters.py +++ b/application/single_app/functions_orchestration_adapters.py @@ -88,6 +88,7 @@ CAPABILITY_DOCUMENT_COMPARE, CAPABILITY_DOCUMENT_SEARCH, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, CAPABILITY_URL_FETCH, CAPABILITY_WEB_SEARCH, @@ -934,13 +935,110 @@ def run_document_compare(step, context, *, settings, user_id, emit, cancel_reque # -------------------------------------------------------------------------------------- -# tabular_merge -> functions_tabular_merge through functions_orchestration_merge +# tabular_inspect and tabular_merge -> functions_tabular_merge through functions_orchestration_merge # -------------------------------------------------------------------------------------- +def _tabular_step_failure(exc, step, user_id, *, failure_code, label): + """The result for a refused or failed spreadsheet step; only catalog text reaches the user.""" + from functions_tabular_merge import TabularMergeError + + if isinstance(exc, TabularMergeError): + failure = build_failure(failure_code(exc)) + log_event( + f'{_LOG_PREFIX} {label} was refused.', + extra={ + 'user_id': user_id, 'step_id': (step or {}).get('step_id'), + 'merge_code': exc.code, 'failure_code': failure['code'], + }, + level=logging.WARNING, + ) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + # Typed authority, cancellation and lifecycle controls belong to the owning runtime. + from functions_orchestration_result_runtime import raise_source_service_failure + + raise_source_service_failure(exc) + log_event( + f'{_LOG_PREFIX} {label} failed.', + extra={'user_id': user_id, 'step_id': (step or {}).get('step_id'), 'error_type': type(exc).__name__}, + level=logging.ERROR, exceptionTraceback=True, + ) + if isinstance(exc, (PermissionError, LookupError)) or type(exc).__name__ in ( + 'ResultUnavailableError', 'DocumentHeldError', + ): + failure = access_failure(exc) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + return _failed_result(f'The {label.lower()} could not be completed.', exc) + + +def run_tabular_inspect(step, context, *, settings, user_id, emit, cancel_requested): + """Describe authorized spreadsheets and how their columns line up; no model and no file.""" + arguments = _arguments(step) + # The merge engine is standard-library only; its Excel readers load only when used. + from functions_tabular_merge import TabularMergeCancelled, TabularMergeError + + try: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect') + input_fingerprint = _internal_result_input_fingerprint(step, context) + from functions_orchestration_merge import ( + build_tabular_merge_sources, + inspection_step_summary, + persist_tabular_inspection_result, + require_tabular_merge_manifest, + tabular_inspect_options, + tabular_merge_limits, + ) + from functions_tabular_merge import inspect_tabular_sources + + with orchestration_file_policy(allow_generated_files=False): + document_ids = _string_list(arguments.get('document_ids')) + limits = tabular_merge_limits(settings) + if len(document_ids) > limits.max_sources: + # A bound source set is checked here; explicit IDs were checked at planning. + raise TabularMergeError('too_many_sources', 'Too many files were selected to inspect here.') + options = tabular_inspect_options(arguments) + service, producer, token, input_fingerprint = _internal_result_context( + step, context, user_id, CAPABILITY_TABULAR_INSPECT, input_fingerprint=input_fingerprint, + ) + manifest = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, + ) + require_tabular_merge_manifest(manifest, document_ids, minimum=1) + _emit(emit, _progress(step, CAPABILITY_TABULAR_INSPECT, 'Inspecting spreadsheets')) + sources = build_tabular_merge_sources( + manifest, user_id, byte_reader=_ctx(context, 'merge_source_reader', None), + ) + inspection = inspect_tabular_sources( + sources, options=options, limits=limits, + cancel_requested=lambda: _is_cancelled(cancel_requested), + ) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect_saving') + task_result = persist_tabular_inspection_result( + service=service, producer=producer, inspection=inspection, sources=manifest, + guard_token=token, input_fingerprint=input_fingerprint, + ) + summary = inspection_step_summary(inspection) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect_finalization') + except (MixedSourceCancellationError, TabularMergeCancelled): + return _cancelled_result('Inspecting spreadsheets was cancelled.') + except Exception as exc: + from functions_orchestration_merge import inspect_failure_code + + return _tabular_step_failure( + exc, step, user_id, failure_code=inspect_failure_code, label='Spreadsheet inspection', + ) + return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) + + def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requested): - """Append the rows of authorized same-structure spreadsheets; no model and no file. + """Append the rows of authorized spreadsheets into one table; no model and no file. - The retained ``records`` output is what ``render_file`` turns into a CSV or XLSX file. + A bound ``mapping`` input is a compose step's prepared column mapping; it is validated + again here and makes the merge keep exactly its columns. The retained ``records`` output + is what ``render_file`` turns into a CSV or XLSX file. """ arguments = _arguments(step) # The merge engine is standard-library only; its Excel readers load only when used. @@ -957,7 +1055,7 @@ def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requeste tabular_merge_limits, tabular_merge_options, ) - from functions_tabular_merge import merge_tabular_sources + from functions_tabular_merge import merge_tabular_sources, tabular_mapping_from_profile with orchestration_file_policy(allow_generated_files=False): document_ids = _string_list(arguments.get('document_ids')) @@ -965,10 +1063,17 @@ def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requeste if len(document_ids) > limits.max_sources: # A bound source set is checked here; explicit IDs were checked at planning. raise TabularMergeError('too_many_sources', 'Too many files were selected to merge here.') - options = tabular_merge_options(arguments) service, producer, token, input_fingerprint = _internal_result_context( step, context, user_id, CAPABILITY_TABULAR_MERGE, input_fingerprint=input_fingerprint, ) + mapping = None + if 'mapping' in (step.get('inputs') or {}): + # Retained-result readers load only when a plan binds a prepared mapping. + from functions_orchestration_result_runtime import read_complete_input, resolve_step_inputs + + readers = resolve_step_inputs(step, context) + mapping = tabular_mapping_from_profile(read_complete_input(readers['mapping'])) + options = tabular_merge_options(arguments, mapping) manifest = resolve_context_source_manifest( context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, ) @@ -984,45 +1089,16 @@ def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requeste raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_saving') task_result = persist_tabular_merge_result( service=service, producer=producer, merge_result=merge_result, sources=manifest, - guard_token=token, input_fingerprint=input_fingerprint, + guard_token=token, input_fingerprint=input_fingerprint, mapping=mapping, ) summary = merge_step_summary(merge_result) raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_finalization') except (MixedSourceCancellationError, TabularMergeCancelled): return _cancelled_result('Merging spreadsheets was cancelled.') - except TabularMergeError as exc: - from functions_orchestration_merge import merge_failure_code - - failure = build_failure(merge_failure_code(exc)) - log_event( - f'{_LOG_PREFIX} Spreadsheet merge was refused.', - extra={ - 'user_id': user_id, 'step_id': (step or {}).get('step_id'), - 'merge_code': exc.code, 'failure_code': failure['code'], - }, - level=logging.WARNING, - ) - return build_step_result( - status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, - ) except Exception as exc: - # Typed authority, cancellation and lifecycle controls belong to the owning runtime. - from functions_orchestration_result_runtime import raise_source_service_failure + from functions_orchestration_merge import merge_failure_code - raise_source_service_failure(exc) - log_event( - f'{_LOG_PREFIX} Spreadsheet merge failed.', - extra={'user_id': user_id, 'step_id': (step or {}).get('step_id'), 'error_type': type(exc).__name__}, - level=logging.ERROR, exceptionTraceback=True, - ) - if isinstance(exc, (PermissionError, LookupError)) or type(exc).__name__ in ( - 'ResultUnavailableError', 'DocumentHeldError', - ): - failure = access_failure(exc) - return build_step_result( - status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, - ) - return _failed_result('The spreadsheet merge could not be completed.', exc) + return _tabular_step_failure(exc, step, user_id, failure_code=merge_failure_code, label='Spreadsheet merge') return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) @@ -1985,6 +2061,7 @@ def run_workflow_results(step, context, *, settings, user_id, emit=None, cancel_ CAPABILITY_DOCUMENT_ANALYZE: run_document_analyze, CAPABILITY_DOCUMENT_COMPARE: run_document_compare, CAPABILITY_TABULAR_ANALYZE: run_tabular_analyze, + CAPABILITY_TABULAR_INSPECT: run_tabular_inspect, CAPABILITY_TABULAR_MERGE: run_tabular_merge, CAPABILITY_WEB_SEARCH: run_web_search, CAPABILITY_URL_FETCH: run_url_fetch, diff --git a/application/single_app/functions_orchestration_deliverables.py b/application/single_app/functions_orchestration_deliverables.py index 0f7a1062e..53969440e 100644 --- a/application/single_app/functions_orchestration_deliverables.py +++ b/application/single_app/functions_orchestration_deliverables.py @@ -41,11 +41,13 @@ CAPABILITY_DOCUMENT_COMPARE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_RENDER_FILE, + CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, MAX_GENERATED_IMAGES_PER_PLAN, + TABULAR_COLUMN_MAPPING_PROFILE, VISUAL_CHART, VISUAL_DIAGRAM, VISUAL_IMAGE_PROPOSAL, @@ -338,6 +340,14 @@ def build_deliverable_availability(settings, *, capabilities, unavailable=None, 'output with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1 ' '(options.sheet_name is optional). Never compose merged rows.' )}) + if CAPABILITY_TABULAR_INSPECT in available and compose: + recipes.append({'for': 'One file merged from CSV or Excel files whose columns differ', 'steps': ( + 'tabular_inspect with the files; compose binding its "inspection" output and declaring ' + 'a structured-v1 output with profile ' + TABULAR_COLUMN_MAPPING_PROFILE + '; ' + 'tabular_merge with the same files and its "mapping" input bound to that output; then ' + 'render_file as above. When the user wants to review the differences first, compose a ' + 'markdown-v1 answer from the inspection instead and merge in a later turn.' + )}) facts.append( 'tabular_merge appends rows exactly; it does not match rows on a key column or change values.' ) diff --git a/application/single_app/functions_orchestration_executor.py b/application/single_app/functions_orchestration_executor.py index b3d4d8440..99ed4e1ad 100644 --- a/application/single_app/functions_orchestration_executor.py +++ b/application/single_app/functions_orchestration_executor.py @@ -63,9 +63,9 @@ ) from functions_orchestration_deliverables import explicit_image_shortfalls from functions_orchestration_registry import ( - CAPABILITY_TABULAR_ANALYZE, CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, - CAPABILITY_WORKFLOW_RUN, DEPENDENCY_PLAN_CONTRACT_VERSION, admitted_export_pairs, external_effect_capability_ids, - get_capability, resolve_available_capability_ids, + CAPABILITY_TABULAR_ANALYZE, CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, + CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, DEPENDENCY_PLAN_CONTRACT_VERSION, admitted_export_pairs, + external_effect_capability_ids, get_capability, resolve_available_capability_ids, ) from functions_orchestration_result_contracts import ( InputBinding, PartialInputNotAcceptedError, ProducerIdentity, ResultContractError, ResultRef, TaskResult, digest, @@ -866,8 +866,10 @@ def _run_dependency_step( readers = resolve_step_inputs(step, context) runtime_step = deepcopy(step) named_sources = [] - # Analyze and Merge accept a search's source set in place of explicit document IDs. - if 'sources' in readers and step['capability_id'] in ('document_analyze', CAPABILITY_TABULAR_MERGE): + # Analyze, Inspect and Merge accept a search's source set in place of explicit document IDs. + if 'sources' in readers and step['capability_id'] in ( + 'document_analyze', CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, + ): named_sources = read_complete_input(readers['sources']) runtime_step['arguments']['document_ids'] = [ source['document_id'] for source in named_sources diff --git a/application/single_app/functions_orchestration_merge.py b/application/single_app/functions_orchestration_merge.py index ef70c3a01..ba25d7bc7 100644 --- a/application/single_app/functions_orchestration_merge.py +++ b/application/single_app/functions_orchestration_merge.py @@ -1,13 +1,16 @@ # functions_orchestration_merge.py -"""Deterministic spreadsheet merging for Gather / Reason / Render orchestration. +"""Deterministic spreadsheet merging and inspection for Gather / Reason / Render orchestration. -Version: 0.261.218 +Version: 0.261.219 +Implemented in: 0.261.218 +Inspection, reconciliation policies and prepared column mappings added in: 0.261.219 -``tabular_merge`` is a Reason capability. Its adapter, ``run_tabular_merge`` in -``functions_orchestration_adapters``, authorizes the sources. This module reads each -original file through the screening-aware byte reader, merges the files with -``functions_tabular_merge`` and retains the merged rows plus a merge report. It creates no -file and calls no model: ``render_file`` delivers a file from the retained ``records``. +``tabular_merge`` is a Reason capability and ``tabular_inspect`` a Gather capability. Their +adapters in ``functions_orchestration_adapters`` authorize the sources. This module reads +each original file through the screening-aware byte reader, merges or inspects the files +with ``functions_tabular_merge``, and retains the results. It creates no file and calls no +model: ``render_file`` delivers a file from the retained ``records``, and a compose step +prepares any column mapping. """ from copy import deepcopy @@ -24,17 +27,18 @@ from functions_orchestration_result_contracts import Completeness, Coverage, RecordColumn, ResultContractError from functions_orchestration_results import NamedOutput from functions_tabular_merge import ( - DEFAULT_SOURCE_COLUMN_NAME, - SCHEMA_POLICY_BY_NAME, + TABULAR_COLUMN_MAPPING_PROFILE, TabularMergeError, TabularMergeLimits, - TabularMergeOptions, TabularMergeSource, + tabular_inspect_options_from_arguments, tabular_merge_format_for_file_name, + tabular_merge_options_from_arguments, ) TABULAR_MERGE_CAPABILITY_ID = "tabular_merge" +TABULAR_INSPECT_CAPABILITY_ID = "tabular_inspect" TABULAR_MERGE_SOURCE_PURPOSE = "native" TABULAR_MERGE_CHECKS = ( "authorized_source_snapshots", @@ -42,13 +46,20 @@ "exact_row_count", "schema_policy_enforced", ) +TABULAR_INSPECT_CHECKS = ( + "authorized_source_snapshots", + "bounded_inspection", +) # Engine codes become application-owned failure codes; their text never reaches a user. MERGE_FAILURE_CODES = { "schema_mismatch": "merge_schema_mismatch", + "nothing_to_merge": "merge_nothing_to_merge", "duplicate_columns": "merge_columns_invalid", "header_too_long": "merge_columns_invalid", "too_many_columns": "merge_columns_invalid", "empty_source": "merge_columns_invalid", + "dedupe_column_not_found": "merge_columns_not_found", + "sort_column_not_found": "merge_columns_not_found", "row_has_extra_values": "merge_rows_invalid", "cell_too_large": "merge_rows_invalid", "unreadable_csv": "merge_source_unreadable", @@ -60,11 +71,25 @@ "duplicate_source": "merge_sources_invalid", "unsupported_format": "merge_sources_invalid", "invalid_source": "merge_sources_invalid", + "invalid_options": "merge_options_invalid", + "invalid_mapping": "merge_mapping_invalid", "too_many_sources": "merge_limit_exceeded", "row_limit_exceeded": "merge_limit_exceeded", + "sort_limit_exceeded": "merge_limit_exceeded", + "too_many_sheets": "merge_limit_exceeded", "source_too_large": "merge_limit_exceeded", "workbook_too_large": "merge_limit_exceeded", } +# Inspection reports unreadable files inside its result; only these refuse the whole step. +INSPECT_FAILURE_CODES = { + "too_few_sources": "inspect_sources_invalid", + "duplicate_source": "inspect_sources_invalid", + "unsupported_format": "inspect_sources_invalid", + "invalid_source": "inspect_sources_invalid", + "invalid_options": "inspect_sources_invalid", + "too_many_sources": "inspect_limit_exceeded", + "inspection_too_large": "inspect_limit_exceeded", +} def merge_failure_code(error): @@ -72,31 +97,38 @@ def merge_failure_code(error): return MERGE_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") +def inspect_failure_code(error): + """The application-owned failure code for an inspection error, or the generic step failure.""" + return INSPECT_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") + + def tabular_merge_limits(settings, *, execution_context=DOCUMENT_ACTION_CONTEXT_CHAT): """The administrator's Merge limits for chat or workflows as engine limits.""" max_sources = get_document_action_max_documents( DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings, ) max_rows = get_document_action_max_rows(DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings) - return TabularMergeLimits(max_sources=max(2, int(max_sources)), max_total_rows=int(max_rows)) + return TabularMergeLimits( + max_sources=max(2, int(max_sources)), max_total_rows=int(max_rows), + max_sort_rows=min(int(max_rows), TabularMergeLimits.max_sort_rows), + ) -def tabular_merge_options(arguments): - """Engine options from validated step arguments.""" - arguments = arguments if isinstance(arguments, dict) else {} - return TabularMergeOptions( - schema_policy=arguments.get("schema_policy") or SCHEMA_POLICY_BY_NAME, - sheet=arguments.get("sheet") or None, - include_source_column=arguments.get("include_source_column", True) is not False, - source_column_name=arguments.get("source_column_name") or DEFAULT_SOURCE_COLUMN_NAME, - ) +def tabular_merge_options(arguments, mapping=None): + """Engine options from validated step arguments and an optional validated mapping.""" + return tabular_merge_options_from_arguments(arguments, mapping=mapping) + + +def tabular_inspect_options(arguments): + """Inspection options from validated step arguments.""" + return tabular_inspect_options_from_arguments(arguments) -def require_tabular_merge_manifest(manifest, document_ids): - """The authorized manifest in merge order, with every source a CSV or Excel file.""" +def require_tabular_merge_manifest(manifest, document_ids, *, minimum=2): + """The authorized manifest in the planned order, with every source a CSV or Excel file.""" document_ids = list(document_ids or ()) if ( - not isinstance(manifest, list) or len(document_ids) < 2 + not isinstance(manifest, list) or len(document_ids) < minimum or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids or any(source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED for source in manifest) ): @@ -132,15 +164,30 @@ def load(source=source): return sources -def persist_tabular_merge_result(*, service, producer, merge_result, sources, guard_token, input_fingerprint): +def merge_report_with_mapping(report, mapping): + """The merge report, recording what a prepared column mapping asked for.""" + if mapping is not None: + report["mapping"] = { + "profile": TABULAR_COLUMN_MAPPING_PROFILE, + "explicitly_ignored": list(mapping["ignored"]), + "low_confidence": deepcopy(mapping["low_confidence"]), + "notes": mapping["notes"], + } + return report + + +def persist_tabular_merge_result( + *, service, producer, merge_result, sources, guard_token, input_fingerprint, mapping=None, +): """Retain the merged rows and the merge report as one complete Reason result.""" if producer.capability_id != TABULAR_MERGE_CAPABILITY_ID: raise ResultContractError("result_merge_invalid") snapshots = analysis_source_snapshot(sources) - columns = tuple(RecordColumn(name, "string") for name in merge_result.columns) + nullable = merge_result.nullable_columns + columns = tuple(RecordColumn(name, "string", name in nullable) for name in merge_result.columns) row_count = merge_result.row_count coverage = Coverage(len(snapshots), len(snapshots), "sources") - report = merge_result.report + report = merge_report_with_mapping(merge_result.report, mapping) limitations = tuple(report.get("limitations") or ()) outputs = [ NamedOutput( @@ -159,10 +206,55 @@ def persist_tabular_merge_result(*, service, producer, merge_result, sources, gu ) +def persist_tabular_inspection_result(*, service, producer, inspection, sources, guard_token, input_fingerprint): + """Retain the inspection as one complete Gather result.""" + if producer.capability_id != TABULAR_INSPECT_CAPABILITY_ID: + raise ResultContractError("result_inspection_invalid") + snapshots = analysis_source_snapshot(sources) + coverage = Coverage(len(snapshots), len(snapshots), "sources") + limitations = list(inspection.get("limitations") or ()) + problems = sum(entry.get("status") == "problem" for entry in inspection.get("sources") or ()) + if problems: + limitations.append(f"{problems} file(s) couldn't be read; the inspection reports why.") + outputs = [ + NamedOutput( + "inspection", "structured-v1", inspection, + Completeness("complete", 1, 1, coverage, "valid", TABULAR_INSPECT_CHECKS, tuple(limitations)), + ), + ] + return service.persist_task_result( + producer=producer, role="gather", status="complete", outputs=outputs, + sources=snapshots, origin="grounded", guard_token=guard_token, input_fingerprint=input_fingerprint, + ) + + def merge_step_summary(merge_result): """A short, application-owned description of a completed merge.""" totals = (merge_result.report or {}).get("totals") or {} - return ( + summary = ( f"Merged {totals.get('rows', merge_result.row_count):,} row(s) from " - f"{totals.get('sources', 0)} file(s) into {totals.get('columns', len(merge_result.columns))} column(s)." + f"{totals.get('sources', 0)} file(s) into {totals.get('columns', len(merge_result.columns))} column(s)" ) + details = [] + if totals.get("duplicates_removed"): + details.append(f"removed {totals['duplicates_removed']:,} duplicate row(s)") + if totals.get("excluded"): + details.append(f"left out {totals['excluded']} file(s) or sheet(s) whose columns don't fit") + return f"{summary}; {'; '.join(details)}." if details else f"{summary}." + + +def inspection_step_summary(inspection): + """A short, application-owned description of a completed inspection.""" + sources = inspection.get("sources") or [] + compatibility = inspection.get("compatibility") or {} + problems = sum(entry.get("status") == "problem" for entry in sources) + if compatibility.get("identical_columns"): + fit = "they all have the same columns" + elif compatibility.get("tables"): + fit = "their columns differ" + else: + fit = "no columns could be read" + summary = f"Inspected {len(sources)} file(s); {fit}" + if problems: + summary += f"; {problems} couldn't be read" + return f"{summary}." diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index 1bdd8d060..8fb1c42b0 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -421,9 +421,10 @@ def resolve_planner_client(settings): invent one. If the user already selected documents, plan around those documents. Use each candidate's server-resolved source_kind, not its display label, to choose compatible work. Never send tabular source IDs to document_analyze or document_compare: those steps -do not admit native tabular inputs. To combine the rows of several tabular sources that share -the same columns into one table or file, use tabular_merge with every one of those sources, never -compose; it appends rows and cannot match rows on a key, so ask when the user may mean that. +do not admit native tabular inputs. To combine the rows of several tabular sources into one +table or file, use tabular_merge with every one of those sources, never compose; it appends +rows and cannot match rows on a key, so ask when the user may mean that. When their columns +may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first. For mixed narrative/tabular comparisons, prepare each source with compatible offered capabilities and compose their named results. Ask each preparation step only for its own sources' contribution, such as values, periods, units and diff --git a/application/single_app/functions_orchestration_registry.py b/application/single_app/functions_orchestration_registry.py index 917e76075..a5a115e58 100644 --- a/application/single_app/functions_orchestration_registry.py +++ b/application/single_app/functions_orchestration_registry.py @@ -43,6 +43,8 @@ from functions_orchestration_result_contracts import ( IMAGE_ASSET_KIND, RESULT_KINDS, ResultContractError, canonical_bytes, ) +# The merge engine imports only the standard library, so the registry stays importable alone. +from functions_tabular_merge import TABULAR_COLUMN_MAPPING_PROFILE # The schema marker every saved plan carries. A plan without it, or with the earlier value, # was written by the removed legacy contract and is never opened or run. @@ -97,6 +99,7 @@ def effective_capability_ids(values): CAPABILITY_DOCUMENT_ANALYZE = 'document_analyze' CAPABILITY_DOCUMENT_COMPARE = 'document_compare' CAPABILITY_TABULAR_ANALYZE = 'tabular_analyze' +CAPABILITY_TABULAR_INSPECT = 'tabular_inspect' CAPABILITY_TABULAR_MERGE = 'tabular_merge' CAPABILITY_WEB_SEARCH = 'web_search' CAPABILITY_URL_FETCH = 'url_fetch' @@ -574,6 +577,27 @@ def generate_image_arguments_schema(options=None): ' This step reads narrative documents; native tabular handoff is not admitted.' ) +# Argument shapes shared by the spreadsheet merge and inspection steps. Descriptors are +# deep-copied whenever they are read, so sharing them here never shares a caller's copy. +_MERGE_COLUMN_NAME_INPUT = {'type': 'string', 'minLength': 1, 'maxLength': 256} +_MERGE_SHEET_INPUT = { + 'type': 'string', + 'minLength': 1, + 'maxLength': 31, + 'description': "One sheet to read in every Excel file. Omit for each workbook's first visible sheet.", +} +_MERGE_SHEETS_INPUT = { + 'type': 'string', + 'enum': ['first', 'all'], + 'description': 'all reads every visible sheet of each workbook; first (default) reads one.', +} +_MERGE_HEADER_ROW_INPUT = { + 'type': 'integer', + 'minimum': 1, + 'maximum': 1000, + 'description': 'The row holding the column headers in every file. Omit to use the first nonblank row.', +} + # The registry itself, in the order a plan tends to read: gather, then reason, then render. # @@ -781,28 +805,103 @@ def generate_image_arguments_schema(options=None): 'max_per_plan': 2, 'adapter': CAPABILITY_TABULAR_ANALYZE, }, + { + 'id': CAPABILITY_TABULAR_INSPECT, + 'label': 'Inspect spreadsheets', + 'role': ROLE_GATHER, + 'request_gate': None, + 'summary': ( + 'Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, ' + 'and how their columns line up.' + ), + 'when_to_use': ( + 'Use before tabular_merge when the files may not share the same columns, or when the user ' + 'asks what sheets, columns or rows CSV or Excel files have. It reports each file\'s headers, ' + 'row counts and sample rows, which files share the same columns, near matches such as ' + '"Customer ID" and "customer_id", a likely title row with the header_row to use, and a ' + 'suggested schema_policy. A file that cannot be read is reported, not fatal. To line up ' + 'differently named columns, bind its "inspection" output to a compose step that declares ' + 'a structured-v1 output with profile ' + TABULAR_COLUMN_MAPPING_PROFILE + ', and bind ' + 'that output as tabular_merge\'s "mapping" input. When the user should review the ' + 'differences first, compose an answer from the inspection and merge in a later turn.' + ), + 'settings_gates': (), + 'settings_gates_any': ( + 'enable_user_workspace', + 'enable_group_workspaces', + 'enable_public_workspaces', + ), + 'gate': _document_action_gate(DOCUMENT_ACTION_TYPE_MERGE), + 'requires_scope': (SCOPE_PERSONAL, SCOPE_GROUP, SCOPE_PUBLIC), + 'inputs': { + 'type': 'object', + 'properties': { + 'document_ids': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1, + 'uniqueItems': True, + 'description': 'The CSV or Excel files to inspect.', + }, + 'doc_scope': { + 'type': 'string', + 'enum': ['all', 'personal', 'group', 'public'], + 'default': 'all', + }, + 'sheet': _MERGE_SHEET_INPUT, + 'sheets': _MERGE_SHEETS_INPUT, + 'header_row': _MERGE_HEADER_ROW_INPUT, + 'sample_rows': { + 'type': 'integer', + 'minimum': 0, + 'maximum': 10, + 'description': 'Sample rows to read per sheet; 3 when omitted.', + }, + }, + 'additionalProperties': False, + }, + 'result_contract_version': 'tabular-inspect-v1', + 'result_outputs': {'inspection': 'structured-v1'}, + # Files a search found can be inspected by binding its source set by name. + 'result_input_kinds': {'sources': ('source-set-v1',)}, + 'partial_inputs_supported': False, + 'produces': (PRODUCES_RETAINED_RESULTS,), + 'cost_class': COST_CLASS_LOW, + 'max_per_plan': 2, + 'adapter': CAPABILITY_TABULAR_INSPECT, + # Inspection serves Merge, so Merge's switch and chat file limit govern it too. + 'document_action_type': DOCUMENT_ACTION_TYPE_MERGE, + }, { 'id': CAPABILITY_TABULAR_MERGE, 'label': 'Merge spreadsheets', 'role': ROLE_REASON, 'request_gate': None, 'summary': ( - 'Combine the rows of two or more CSV or Excel files that share the same columns into ' - 'one table, exactly and without a model.' + 'Combine the rows of two or more CSV or Excel files into one table, exactly and without ' + 'a model, lining up columns by name, alias or a prepared mapping.' ), 'when_to_use': ( 'Use when the user asks to merge, combine, append, stack or consolidate several CSV or ' - 'Excel files with the same columns into one table or file. Every row is kept exactly, in ' - 'the order the files are listed. It never matches rows on a key column (no joins or ' - 'lookups) and never edits values; when "merge" could mean matching rows on a key, ask. ' - 'Name the files in document_ids in merge order, or bind inputs.sources to a search ' - "step's sources output when the files must be found first. Columns are matched by name " - 'in any order; schema_policy exact_order also requires the same order. sheet picks one ' - "sheet in every workbook; otherwise each workbook's first visible sheet is read. " - 'include_source_column adds each row\'s file name. To deliver a file, bind the records ' - 'output to render_file with csv and exact_tabular_records_v1, or xlsx and ' - 'exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the ' - 'chat limit, or one that should repeat, belongs in a workflow.' + 'Excel files into one table or file. Every row is kept, in the order the files are listed. ' + 'It never matches rows on a key column (no joins or lookups) and never edits values; when ' + '"merge" could mean matching rows on a key, ask. Name the files in document_ids in merge ' + "order, or bind inputs.sources to a search step's sources output when the files must be " + 'found first. schema_policy: by_name (default) needs the same columns in any order, ' + 'exact_order also the same order, union keeps every column from every file and leaves a ' + 'missing one blank, mapped keeps exactly "columns". column_aliases lines up other header ' + 'names, e.g. {"Customer ID": ["cust_id"]}. When the columns may differ and how they line ' + 'up is unclear, run tabular_inspect first and bind a compose step\'s ' + + TABULAR_COLUMN_MAPPING_PROFILE + ' output as the "mapping" input, without columns or ' + 'aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit ' + 'instead of failing. header_row skips title rows above the headers. sheet picks one sheet; ' + 'sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes ' + 'duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last ' + '(dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds ' + 'each row\'s file name. To deliver a file, bind the records output to render_file with ' + 'csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose ' + 'merged rows. A merge of more files than the chat limit, or one that should repeat, ' + 'belongs in a workflow.' ), 'settings_gates': (), 'settings_gates_any': ( @@ -829,19 +928,36 @@ def generate_image_arguments_schema(options=None): }, 'schema_policy': { 'type': 'string', - 'enum': ['by_name', 'exact_order'], + 'enum': ['by_name', 'exact_order', 'union', 'mapped'], 'default': 'by_name', 'description': ( - 'by_name accepts the same columns in any order; exact_order also requires ' - 'the same order.' + 'by_name: same columns in any order. exact_order: same columns in the same ' + 'order. union: every column from every file. mapped: exactly "columns".' ), }, - 'sheet': { - 'type': 'string', - 'minLength': 1, - 'maxLength': 31, - 'description': "The sheet to read in every Excel file. Omit for each workbook's first visible sheet.", + 'columns': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 255, + 'uniqueItems': True, + 'description': 'mapped only: the merged columns, in order.', }, + 'column_aliases': { + 'type': 'object', + 'maxProperties': 256, + 'additionalProperties': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 32, + 'uniqueItems': True, + }, + 'description': 'Each column name mapped to the other header names that mean it.', + }, + 'sheet': _MERGE_SHEET_INPUT, + 'sheets': _MERGE_SHEETS_INPUT, + 'header_row': _MERGE_HEADER_ROW_INPUT, 'include_source_column': { 'type': 'boolean', 'default': True, @@ -853,13 +969,52 @@ def generate_image_arguments_schema(options=None): 'maxLength': 128, 'default': 'Source File', }, + 'on_incompatible': { + 'type': 'string', + 'enum': ['fail', 'exclude'], + 'description': 'fail (default) merges nothing when a file does not fit; exclude leaves it out.', + }, + 'dedupe': { + 'type': 'string', + 'enum': ['none', 'exact_rows', 'key_columns'], + 'description': 'Remove duplicate rows; none when omitted.', + }, + 'dedupe_columns': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 16, + 'uniqueItems': True, + 'description': 'key_columns only: the columns that identify a duplicate.', + }, + 'dedupe_keep': { + 'type': 'string', + 'enum': ['first', 'last'], + 'description': 'Which duplicate to keep; first when omitted.', + }, + 'sort_by': { + 'type': 'array', + 'maxItems': 3, + 'items': { + 'type': 'object', + 'properties': { + 'column': _MERGE_COLUMN_NAME_INPUT, + 'descending': {'type': 'boolean'}, + 'value_type': {'type': 'string', 'enum': ['text', 'number', 'date']}, + }, + 'required': ['column'], + 'additionalProperties': False, + }, + 'description': 'Sort keys, most significant first; text ascending when not stated.', + }, }, 'additionalProperties': False, }, 'result_contract_version': 'tabular-merge-v1', 'result_outputs': {'records': 'records-v1', 'report': 'structured-v1'}, - # Files a search found can be merged by binding its source set by name. - 'result_input_kinds': {'sources': ('source-set-v1',)}, + # Files a search found can be merged by binding its source set by name, and a compose + # step's prepared column mapping lines up differently named columns. + 'result_input_kinds': {'sources': ('source-set-v1',), 'mapping': ('structured-v1',)}, 'partial_inputs_supported': False, 'produces': (PRODUCES_RETAINED_RESULTS,), 'cost_class': COST_CLASS_MEDIUM, diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index 576323f79..f3cda6a99 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -56,6 +56,7 @@ CAPABILITY_COMPOSE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, @@ -75,6 +76,11 @@ ResultContractError, StepBindings, TaskResult, canonical_bytes, output_name, validate_input_bindings, ) from functions_orchestration_deliverables import DeliverableError, compile_deliverables +# The merge engine imports only the standard library; plan validation shares its option rules. +from functions_tabular_merge import ( + TABULAR_COLUMN_MAPPING_PROFILE, TabularMergeError, tabular_inspect_options_from_arguments, + tabular_merge_options_from_arguments, +) ORCHESTRATION_ELICITATION_CONTRACT_VERSION = 2 @@ -571,6 +577,37 @@ def _apply_image_input_policy(steps, existing_results): ) +def _require_merge_mapping_producers(steps): + """A merge's mapping input must come from a compose step that prepared a column mapping. + + A retained result named by alias is checked against the profile when the merge runs. + """ + by_id = {step['step_id']: step for step in steps} + for step in steps: + if step['capability_id'] != CAPABILITY_TABULAR_MERGE: + continue + for spec in step_input_specs(step): + if spec.name != 'mapping' or spec.binding.step_id is None: + continue + producer = by_id.get(spec.binding.step_id) + if producer is None: + # A missing producer is reported by the binding validation that follows. + continue + output = next( + (item for item in producer['outputs'] if item.get('name') == spec.binding.output_name), None, + ) + if ( + producer['capability_id'] != CAPABILITY_COMPOSE + or (output or {}).get('profile') != TABULAR_COLUMN_MAPPING_PROFILE + ): + raise PlanValidationError( + f'Step "{step["step_id"]}" binds a "mapping" input that is not a prepared column ' + 'mapping. Bind the output of a compose step declared as structured-v1 with profile ' + f'{TABULAR_COLUMN_MAPPING_PROFILE}.', + code='mapping_profile_required', rule='merge_mapping_profile', + ) + + def _reject_workflow_proposal_consumers(steps, final_response): """A workflow proposal is reviewed on its own card, so no step and no answer may read it.""" producers = {step['step_id'] for step in steps if step['capability_id'] == CAPABILITY_WORKFLOW_PROPOSE} @@ -881,9 +918,35 @@ def validate_dependency_plan( ) if arguments.get('document_ids') and 'sources' in step['inputs']: raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + try: + tabular_merge_options_from_arguments(arguments, mapping_bound='mapping' in step['inputs']) + except TabularMergeError as exc: + raise PlanValidationError( + f'Step "{step["step_id"]}" has merge settings that cannot be used together: ' + f'{exc.message}', + code='merge_options_invalid', rule='merge_options_invalid', + ) from exc + if capability_id == CAPABILITY_TABULAR_INSPECT: + if not arguments.get('document_ids') and 'sources' not in step['inputs']: + raise PlanValidationError( + 'Inspect requires arguments.document_ids naming the CSV or Excel files, or an ' + 'inputs.sources source-set binding.', + code='source_binding_required', rule='inspect_sources_required', + ) + if arguments.get('document_ids') and 'sources' in step['inputs']: + raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + try: + tabular_inspect_options_from_arguments(arguments) + except TabularMergeError as exc: + raise PlanValidationError( + f'Step "{step["step_id"]}" has inspection settings that cannot be used together: ' + f'{exc.message}', + code='merge_options_invalid', rule='inspect_options_invalid', + ) from exc accepted.append(step) if not any(step['enabled'] for step in accepted): raise PlanValidationError('The plan contains no enabled work.') + _require_merge_mapping_producers(accepted) _apply_image_input_policy(accepted, existing_results) _reject_workflow_proposal_consumers(accepted, plan.get('final_response')) _reject_workflow_run_consumers(accepted, plan.get('final_response')) @@ -1017,14 +1080,17 @@ def validate_plan_document_source_kinds(plan, document_source_kinds): if not isinstance(document_source_kinds, dict): raise PlanValidationError('Document type metadata is unavailable.', code='source_metadata_invalid') for step in plan.get('steps') or []: - if step.get('enabled', True) and step.get('capability_id') == CAPABILITY_TABULAR_MERGE: + if step.get('enabled', True) and step.get('capability_id') in ( + CAPABILITY_TABULAR_MERGE, CAPABILITY_TABULAR_INSPECT, + ): if any( document_source_kinds.get(document_id) not in (None, 'tabular') for document_id in plan_document_ids({'steps': [step]}) ): + action = 'merges' if step['capability_id'] == CAPABILITY_TABULAR_MERGE else 'inspects' raise PlanValidationError( - f'Step "{step["step_id"]}" merges only CSV or Excel files, and a selected source is ' - 'another kind of document. Merge only the tabular sources, or prepare the others ' + f'Step "{step["step_id"]}" {action} only CSV or Excel files, and a selected source is ' + 'another kind of document. Use only the tabular sources there, or prepare the others ' 'with compatible offered capabilities. Keep every selected source.', code='source_kind_invalid', rule='tabular_source_required', ) @@ -1532,13 +1598,22 @@ def summarize_plan(plan): 'image_request_invalid': 'The image model did not accept the planned image request.', 'retry_would_repeat': 'A retry would send the same request that was declined. Ask again with a different description instead.', 'merge_schema_mismatch': ( - "The selected files don't all have the same columns, so nothing was merged. Ask to " - 'inspect their columns to see how they differ.' + "The selected files don't all have the same columns, so nothing was merged. Ask to keep every " + 'column, to line up columns that have different names, or to leave out the files that ' + "don't match." + ), + 'merge_nothing_to_merge': ( + 'None of the selected files had a sheet with column headers that fit the merge, so nothing ' + 'was merged.' ), 'merge_columns_invalid': ( 'A selected file has duplicate or overly long column headers, no header row, or too many ' 'columns, so nothing was merged.' ), + 'merge_columns_not_found': ( + "A column named for removing duplicates or sorting isn't in the merged files, so nothing " + 'was merged.' + ), 'merge_rows_invalid': ( 'A selected file has a row with more values than column headers, or a value longer than a ' 'spreadsheet cell allows, so nothing was merged.' @@ -1549,10 +1624,17 @@ def summarize_plan(plan): ), 'merge_sheet_not_found': "A selected workbook doesn't have the requested sheet, so nothing was merged.", 'merge_sources_invalid': 'Merging needs at least two different CSV or Excel files (.csv, .xlsx, .xlsm or .xls).', + 'merge_options_invalid': "This merge's settings can't be used together, so nothing was merged.", + 'merge_mapping_invalid': "The column mapping prepared for this merge isn't valid, so nothing was merged.", 'merge_limit_exceeded': ( 'The merge is larger than chat allows. Merge fewer or smaller files, or ask for a workflow ' 'to merge them.' ), + 'inspect_sources_invalid': 'Only CSV and Excel files (.csv, .xlsx, .xlsm or .xls) can be inspected.', + 'inspect_limit_exceeded': ( + 'The selected files have too many sheets or columns to inspect here. Inspect fewer files at ' + 'a time.' + ), 'step_failed': 'This operation could not complete.', 'message_not_saved': 'The explanation could not be saved. Reload this run to check its durable status.', LEGACY_PLAN_CODE: LEGACY_PLAN_MESSAGE, diff --git a/application/single_app/functions_orchestration_services.py b/application/single_app/functions_orchestration_services.py index ffd0a2382..7ec64866b 100644 --- a/application/single_app/functions_orchestration_services.py +++ b/application/single_app/functions_orchestration_services.py @@ -28,6 +28,9 @@ InputBinding, ResultContractError, ResultRef, TaskResult, canonical_digest, ) from functions_orchestration_results import OrchestrationResultAccess, OrchestrationResults +from functions_tabular_merge import ( + TABULAR_COLUMN_MAPPING_PROFILE, TabularMergeError, tabular_column_mapping_schema, tabular_mapping_from_profile, +) from functions_workflow_result_store import _orchestration_identity @@ -35,11 +38,24 @@ def composition_profiles(): - """Use the renderer's exact prepared-content schema, not a second slide grammar.""" - return {PREPARED_SLIDE_DECK_VERSION: get_prepared_slide_deck_schema()} + """Use each owner's exact prepared-content schema, not a second grammar. + + The slide deck schema is the renderer's; the column mapping schema is the merge engine's. + """ + return { + PREPARED_SLIDE_DECK_VERSION: get_prepared_slide_deck_schema(), + TABULAR_COLUMN_MAPPING_PROFILE: tabular_column_mapping_schema(), + } def validate_composition_profile(profile, value): + if profile == TABULAR_COLUMN_MAPPING_PROFILE: + try: + tabular_mapping_from_profile(value) + except TabularMergeError as exc: + # Asking once more names the broken rule; the reply itself is never logged. + raise ResultContractError("result_schema_invalid") from exc + return True if profile != PREPARED_SLIDE_DECK_VERSION: raise ResultContractError("result_profile_unavailable") # Office libraries are execution dependencies, not registry/bootstrap dependencies. diff --git a/application/single_app/functions_tabular_merge.py b/application/single_app/functions_tabular_merge.py index ecb18e4e3..c011977c1 100644 --- a/application/single_app/functions_tabular_merge.py +++ b/application/single_app/functions_tabular_merge.py @@ -1,33 +1,77 @@ # functions_tabular_merge.py -"""Deterministic merging of same-structure CSV and Excel files into one table. +"""Deterministic merging and inspection of CSV and Excel files. -Version: 0.261.218 +Version: 0.261.219 +Implemented in: 0.261.218 +Reconciliation policies, sheet modes, duplicate removal, sorting and inspection added in: 0.261.219 The engine is pure: it receives already-authorized byte loaders, never resolves documents, settings, storage, routes, or models, and performs no model work. Values are kept as text so leading zeros, codes, and identifiers survive exactly. Rows are spooled to a bounded temporary file during the single parse, so callers learn the exact row count before they persist or render the merged result. + +Schema policies decide which columns the merged table has: + +- ``by_name``: every file has the same columns, in any order (the first file's spelling is kept). +- ``exact_order``: every file has the same columns in the same order. +- ``union``: every column from every file; a file without a column leaves it blank (null). +- ``mapped``: exactly the requested ``columns``; other columns are left out and reported. + +``column_aliases`` rename alternative headers to one target name before matching, under +every policy. Files whose columns do not fit fail the merge, or are left out and reported +when ``on_incompatible`` is ``exclude``. """ +import bisect import codecs import csv +from array import array from dataclasses import dataclass, field -from datetime import date, datetime, time, timedelta +from datetime import date, datetime, time, timedelta, timezone +import hashlib import io import json +import math import os +import re import tempfile -from typing import Callable, Iterator, List, Optional, Sequence, Tuple +from typing import Callable, List, Mapping, Optional, Sequence, Tuple import unicodedata import zipfile TABULAR_MERGE_REPORT_VERSION = "tabular-merge-report-v1" +TABULAR_INSPECTION_VERSION = "tabular-inspection-v1" SCHEMA_POLICY_BY_NAME = "by_name" SCHEMA_POLICY_EXACT_ORDER = "exact_order" -TABULAR_MERGE_SCHEMA_POLICIES = (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER) +SCHEMA_POLICY_UNION = "union" +SCHEMA_POLICY_MAPPED = "mapped" +TABULAR_MERGE_SCHEMA_POLICIES = ( + SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER, SCHEMA_POLICY_UNION, SCHEMA_POLICY_MAPPED, +) + +SHEETS_FIRST = "first" +SHEETS_ALL = "all" +TABULAR_MERGE_SHEET_MODES = (SHEETS_FIRST, SHEETS_ALL) + +ON_INCOMPATIBLE_FAIL = "fail" +ON_INCOMPATIBLE_EXCLUDE = "exclude" +TABULAR_MERGE_INCOMPATIBLE_MODES = (ON_INCOMPATIBLE_FAIL, ON_INCOMPATIBLE_EXCLUDE) + +DEDUPE_NONE = "none" +DEDUPE_EXACT_ROWS = "exact_rows" +DEDUPE_KEY_COLUMNS = "key_columns" +TABULAR_MERGE_DEDUPE_MODES = (DEDUPE_NONE, DEDUPE_EXACT_ROWS, DEDUPE_KEY_COLUMNS) +DEDUPE_KEEP_FIRST = "first" +DEDUPE_KEEP_LAST = "last" +TABULAR_MERGE_DEDUPE_KEEP = (DEDUPE_KEEP_FIRST, DEDUPE_KEEP_LAST) + +SORT_TYPE_TEXT = "text" +SORT_TYPE_NUMBER = "number" +SORT_TYPE_DATE = "date" +TABULAR_MERGE_SORT_TYPES = (SORT_TYPE_TEXT, SORT_TYPE_NUMBER, SORT_TYPE_DATE) TABULAR_MERGE_FORMAT_CSV = "csv" TABULAR_MERGE_FORMAT_XLSX = "xlsx" @@ -41,12 +85,24 @@ TABULAR_MERGE_SUPPORTED_EXTENSIONS = tuple(TABULAR_MERGE_EXTENSION_FORMATS) DEFAULT_SOURCE_COLUMN_NAME = "Source File" +DEFAULT_SHEET_COLUMN_NAME = "Source Sheet" MAX_SOURCE_COLUMN_NAME_CHARS = 128 MAX_SHEET_NAME_CHARS = 31 +MAX_COLUMN_NAME_BYTES = 256 +MAX_HEADER_ROW = 1000 +MAX_MAPPED_COLUMNS = 255 +MAX_ALIAS_TARGETS = 256 +MAX_ALIASES_PER_TARGET = 32 +MAX_DEDUPE_COLUMNS = 16 +MAX_SORT_COLUMNS = 3 +DEFAULT_INSPECT_SAMPLE_ROWS = 3 +MAX_INSPECT_SAMPLE_ROWS = 10 SOURCE_STATUS_MERGED = "merged" SOURCE_STATUS_COMPATIBLE = "compatible" SOURCE_STATUS_SCHEMA_MISMATCH = "schema_mismatch" +SOURCE_STATUS_EXCLUDED = "excluded" +SOURCE_STATUS_SKIPPED = "skipped" SOURCE_STATUS_NOT_PROCESSED = "not_processed" _CSV_DELIMITERS = (",", ";", "\t", "|") @@ -56,12 +112,30 @@ _ZIP_MAGIC = b"PK\x03\x04" _OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" _MAX_EXACT_INTEGER_FLOAT = float(2 ** 53) +_INSPECT_CELL_CHARS = 80 +_INSPECT_MAX_BYTES = 96 * 1024 +_NUMBER_TEXT = re.compile( + r"[+-]?(?:(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?|\.\d+)(?:[eE][+-]?\d+)?" +) +_CURRENCY_PREFIXES = "$\u20ac\u00a3\u00a5" +_NOT_WORD = re.compile(r"[\W_]+") _EXCEL_LIMITATIONS = ( "Excel formulas are merged as their last calculated values; a workbook saved without " "calculated values merges those cells as blanks.", "Excel numbers, booleans and dates are written as plain text, with dates in ISO 8601 form.", ) +_EXCLUDED_LIMITATION = ( + "{count} of {total} file(s) or sheet(s) were left out because their columns don't fit the " + "merge; the merge report lists each one and why." +) +_BLANK_COLUMNS_LIMITATION = ( + "Columns that some files don't have are left blank for those files' rows; the merge report " + "lists the missing columns." +) +_IGNORED_COLUMNS_LIMITATION = ( + "Columns that aren't in the requested column list were left out; the merge report lists them." +) class TabularMergeError(ValueError): @@ -94,12 +168,14 @@ class TabularMergeLimits: max_cell_chars: int = 32_767 max_workbook_uncompressed_bytes: int = 512 * 1024 * 1024 max_workbook_entries: int = 10_000 + max_sort_rows: int = 250_000 + max_sheets_per_source: int = 100 def __post_init__(self): for name in ( "max_sources", "max_total_rows", "max_source_bytes", "max_columns", "max_header_bytes", "max_cell_chars", "max_workbook_uncompressed_bytes", - "max_workbook_entries", + "max_workbook_entries", "max_sort_rows", "max_sheets_per_source", ): value = getattr(self, name) if type(value) is not int or value < 1: @@ -108,14 +184,47 @@ def __post_init__(self): raise ValueError("max_sources must allow at least two sources.") +@dataclass(frozen=True) +class TabularMergeSort: + """One sort key: a merged column, its direction and how its values compare.""" + + column: str + descending: bool = False + value_type: str = SORT_TYPE_TEXT + + def __post_init__(self): + name = _display_header(self.column) if isinstance(self.column, str) else "" + if not name or len(name.encode("utf-8")) > MAX_COLUMN_NAME_BYTES: + raise TabularMergeError("invalid_options", "Each sort key needs a column name.") + object.__setattr__(self, "column", name) + if type(self.descending) is not bool: + raise TabularMergeError("invalid_options", "A sort direction must be true or false.") + if self.value_type not in TABULAR_MERGE_SORT_TYPES: + raise TabularMergeError( + "invalid_options", f"A sort type must be one of: {', '.join(TABULAR_MERGE_SORT_TYPES)}.", + ) + + def to_dict(self): + return {"column": self.column, "descending": self.descending, "value_type": self.value_type} + + @dataclass(frozen=True) class TabularMergeOptions: - """How sources are matched and how provenance is recorded.""" + """How sources are matched, combined, de-duplicated, ordered and attributed.""" schema_policy: str = SCHEMA_POLICY_BY_NAME sheet: Optional[str] = None include_source_column: bool = True source_column_name: str = DEFAULT_SOURCE_COLUMN_NAME + sheets: str = SHEETS_FIRST + header_row: Optional[int] = None + columns: Tuple[str, ...] = () + column_aliases: Tuple[Tuple[str, Tuple[str, ...]], ...] = () + on_incompatible: str = ON_INCOMPATIBLE_FAIL + dedupe: str = DEDUPE_NONE + dedupe_columns: Tuple[str, ...] = () + dedupe_keep: str = DEDUPE_KEEP_FIRST + sort_by: Tuple[TabularMergeSort, ...] = () def __post_init__(self): if self.schema_policy not in TABULAR_MERGE_SCHEMA_POLICIES: @@ -123,13 +232,7 @@ def __post_init__(self): "invalid_options", f"Schema policy must be one of: {', '.join(TABULAR_MERGE_SCHEMA_POLICIES)}.", ) - if self.sheet is not None: - if not isinstance(self.sheet, str) or not self.sheet.strip(): - raise TabularMergeError("invalid_options", "A sheet name must be nonempty text.") - if len(self.sheet) > MAX_SHEET_NAME_CHARS: - raise TabularMergeError( - "invalid_options", f"A sheet name has at most {MAX_SHEET_NAME_CHARS} characters.", - ) + _validate_table_selection(self.sheet, self.sheets, self.header_row) if type(self.include_source_column) is not bool: raise TabularMergeError("invalid_options", "include_source_column must be true or false.") name = _display_header(self.source_column_name) if isinstance(self.source_column_name, str) else "" @@ -139,6 +242,89 @@ def __post_init__(self): f"The source column name must be 1 to {MAX_SOURCE_COLUMN_NAME_CHARS} characters.", ) + columns = _column_names(self.columns, "The requested columns", maximum=MAX_MAPPED_COLUMNS) + object.__setattr__(self, "columns", columns) + if self.schema_policy == SCHEMA_POLICY_MAPPED and not columns: + raise TabularMergeError("invalid_options", "The mapped policy needs the list of columns to keep.") + if self.schema_policy != SCHEMA_POLICY_MAPPED and columns: + raise TabularMergeError("invalid_options", "A list of columns is used only by the mapped policy.") + + aliases = _alias_pairs(self.column_aliases) + object.__setattr__(self, "column_aliases", aliases) + if self.schema_policy == SCHEMA_POLICY_MAPPED: + targets = {normalize_header_name(column) for column in columns} + if any(normalize_header_name(target) not in targets for target, _ in aliases): + raise TabularMergeError( + "invalid_options", "Every alias must rename a header to one of the requested columns.", + ) + + if self.on_incompatible not in TABULAR_MERGE_INCOMPATIBLE_MODES: + raise TabularMergeError( + "invalid_options", + f"on_incompatible must be one of: {', '.join(TABULAR_MERGE_INCOMPATIBLE_MODES)}.", + ) + if self.dedupe not in TABULAR_MERGE_DEDUPE_MODES: + raise TabularMergeError( + "invalid_options", f"dedupe must be one of: {', '.join(TABULAR_MERGE_DEDUPE_MODES)}.", + ) + if self.dedupe_keep not in TABULAR_MERGE_DEDUPE_KEEP: + raise TabularMergeError( + "invalid_options", f"dedupe_keep must be one of: {', '.join(TABULAR_MERGE_DEDUPE_KEEP)}.", + ) + dedupe_columns = _column_names( + self.dedupe_columns, "The duplicate key columns", maximum=MAX_DEDUPE_COLUMNS, + ) + object.__setattr__(self, "dedupe_columns", dedupe_columns) + if self.dedupe == DEDUPE_KEY_COLUMNS and not dedupe_columns: + raise TabularMergeError("invalid_options", "Removing duplicates by key needs the key columns.") + if self.dedupe != DEDUPE_KEY_COLUMNS and dedupe_columns: + raise TabularMergeError("invalid_options", "Key columns are used only when removing duplicates by key.") + + sort_by = tuple( + item if isinstance(item, TabularMergeSort) else _sort_from_mapping(item) + for item in (self.sort_by or ()) + ) + if len(sort_by) > MAX_SORT_COLUMNS: + raise TabularMergeError("invalid_options", f"Sort by at most {MAX_SORT_COLUMNS} columns.") + if len({normalize_header_name(item.column) for item in sort_by}) != len(sort_by): + raise TabularMergeError("invalid_options", "Each column can be sorted on only once.") + object.__setattr__(self, "sort_by", sort_by) + + def policy_report(self): + """The options as plain data for the merge report.""" + return { + "schema_policy": self.schema_policy, + "sheet": self.sheet, + "sheets": self.sheets, + "header_row": self.header_row, + "include_source_column": self.include_source_column, + "source_column_name": _display_header(self.source_column_name) if self.include_source_column else None, + "columns": list(self.columns), + "column_aliases": {target: list(aliases) for target, aliases in self.column_aliases}, + "on_incompatible": self.on_incompatible, + "dedupe": self.dedupe, + "dedupe_columns": list(self.dedupe_columns), + "dedupe_keep": self.dedupe_keep if self.dedupe != DEDUPE_NONE else None, + "sort_by": [item.to_dict() for item in self.sort_by], + } + + +@dataclass(frozen=True) +class TabularInspectOptions: + """Which sheets and header row to inspect, and how many sample rows to keep.""" + + sheet: Optional[str] = None + sheets: str = SHEETS_FIRST + header_row: Optional[int] = None + sample_rows: int = DEFAULT_INSPECT_SAMPLE_ROWS + + def __post_init__(self): + _validate_table_selection(self.sheet, self.sheets, self.header_row) + if type(self.sample_rows) is not int or not 0 <= self.sample_rows <= MAX_INSPECT_SAMPLE_ROWS: + raise TabularMergeError( + "invalid_options", f"Sample rows must be a whole number from 0 to {MAX_INSPECT_SAMPLE_ROWS}.", + ) + @dataclass(frozen=True) class TabularMergeSource: @@ -156,12 +342,294 @@ def resolved_extension(self): return extension +def _validate_table_selection(sheet, sheets, header_row): + if sheet is not None: + if not isinstance(sheet, str) or not sheet.strip(): + raise TabularMergeError("invalid_options", "A sheet name must be nonempty text.") + if len(sheet) > MAX_SHEET_NAME_CHARS: + raise TabularMergeError( + "invalid_options", f"A sheet name has at most {MAX_SHEET_NAME_CHARS} characters.", + ) + if sheets not in TABULAR_MERGE_SHEET_MODES: + raise TabularMergeError( + "invalid_options", f"sheets must be one of: {', '.join(TABULAR_MERGE_SHEET_MODES)}.", + ) + if sheet is not None and sheets == SHEETS_ALL: + raise TabularMergeError("invalid_options", "Name one sheet, or read all sheets, but not both.") + if header_row is not None and (type(header_row) is not int or not 1 <= header_row <= MAX_HEADER_ROW): + raise TabularMergeError( + "invalid_options", f"The header row must be a whole number from 1 to {MAX_HEADER_ROW}.", + ) + + +def _column_name(value, label): + name = _display_header(value) if isinstance(value, str) else "" + if not name or len(name.encode("utf-8")) > MAX_COLUMN_NAME_BYTES: + raise TabularMergeError( + "invalid_options", f"{label} must be nonempty names of at most {MAX_COLUMN_NAME_BYTES} bytes.", + ) + return name + + +def _column_names(values, label, *, maximum): + if values is None: + return () + if isinstance(values, (str, bytes)) or not isinstance(values, (list, tuple)): + raise TabularMergeError("invalid_options", f"{label} must be a list of column names.") + names = tuple(_column_name(value, label) for value in values) + if len(names) > maximum: + raise TabularMergeError("invalid_options", f"{label} can name at most {maximum} columns.") + if len({name.casefold() for name in names}) != len(names): + raise TabularMergeError("invalid_options", f"{label} name a column more than once.") + return names + + +def _alias_pairs(value): + """Validated ``((target, (alias, ...)), ...)``; identity aliases are dropped.""" + if not value: + return () + if isinstance(value, Mapping): + items = list(value.items()) + elif isinstance(value, (list, tuple)): + items = list(value) + else: + raise TabularMergeError("invalid_options", "Column aliases must map each column to its other names.") + if len(items) > MAX_ALIAS_TARGETS: + raise TabularMergeError("invalid_options", f"Column aliases can rename at most {MAX_ALIAS_TARGETS} columns.") + pairs = [] + targets = {} + for item in items: + if not isinstance(item, (list, tuple)) or len(item) != 2: + raise TabularMergeError("invalid_options", "Column aliases must map each column to its other names.") + target, aliases = item + target = _column_name(target, "Alias targets") + if target.casefold() in targets: + raise TabularMergeError("invalid_options", "Column aliases name a target column more than once.") + if isinstance(aliases, (str, bytes)) or not isinstance(aliases, (list, tuple)) or not aliases: + raise TabularMergeError("invalid_options", "Each aliased column needs a list of its other names.") + if len(aliases) > MAX_ALIASES_PER_TARGET: + raise TabularMergeError( + "invalid_options", f"A column can have at most {MAX_ALIASES_PER_TARGET} other names.", + ) + names = [] + for alias in aliases: + name = _column_name(alias, "Column aliases") + if name.casefold() != target.casefold() and name.casefold() not in {item.casefold() for item in names}: + names.append(name) + targets[target.casefold()] = names + pairs.append((target, tuple(names))) + seen = {} + for target, names in pairs: + for name in names: + key = name.casefold() + if key in targets: + raise TabularMergeError( + "invalid_options", f'"{name}" is both a column name and another column\'s alias.', + ) + if key in seen and seen[key] != target.casefold(): + raise TabularMergeError("invalid_options", f'"{name}" is an alias of more than one column.') + seen[key] = target.casefold() + return tuple((target, names) for target, names in pairs if names) + + +def _sort_from_mapping(value): + if not isinstance(value, Mapping): + raise TabularMergeError("invalid_options", "Each sort key must name a column.") + unknown = set(value) - {"column", "descending", "value_type"} + if unknown: + raise TabularMergeError("invalid_options", "A sort key has an unsupported setting.") + return TabularMergeSort( + column=value.get("column"), + descending=value.get("descending", False), + value_type=value.get("value_type", SORT_TYPE_TEXT), + ) + + def tabular_merge_format_for_file_name(file_name): """The merge reader format for a file name, or None when unsupported.""" extension = os.path.splitext(str(file_name or ""))[1].strip().lower() return TABULAR_MERGE_EXTENSION_FORMATS.get(extension) +def tabular_merge_options_from_arguments(arguments, *, mapping=None, mapping_bound=False): + """Engine options from plan arguments; a bound column mapping makes the policy ``mapped``. + + At planning time the mapping is not prepared yet, so ``mapping_bound`` checks that the + other arguments can be combined with one. Invalid combinations raise ``invalid_options``. + """ + arguments = arguments if isinstance(arguments, Mapping) else {} + policy = arguments.get("schema_policy") or SCHEMA_POLICY_BY_NAME + columns = arguments.get("columns") or () + aliases = arguments.get("column_aliases") or () + if mapping is not None or mapping_bound: + if columns or aliases: + raise TabularMergeError( + "invalid_options", "Use either a bound column mapping or explicit columns and aliases, not both.", + ) + if policy not in (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_MAPPED): + raise TabularMergeError("invalid_options", "A bound column mapping uses the mapped schema policy.") + policy = SCHEMA_POLICY_MAPPED + if mapping is None: + # The prepared mapping supplies the real columns when the merge runs. + columns = ("Mapped column",) + else: + columns = mapping["columns"] + aliases = mapping["column_aliases"] + return TabularMergeOptions( + schema_policy=policy, + sheet=arguments.get("sheet") or None, + include_source_column=arguments.get("include_source_column", True) is not False, + source_column_name=arguments.get("source_column_name") or DEFAULT_SOURCE_COLUMN_NAME, + sheets=arguments.get("sheets") or SHEETS_FIRST, + header_row=arguments.get("header_row"), + columns=columns, + column_aliases=aliases, + on_incompatible=arguments.get("on_incompatible") or ON_INCOMPATIBLE_FAIL, + dedupe=arguments.get("dedupe") or DEDUPE_NONE, + dedupe_columns=arguments.get("dedupe_columns") or (), + dedupe_keep=arguments.get("dedupe_keep") or DEDUPE_KEEP_FIRST, + sort_by=arguments.get("sort_by") or (), + ) + + +def tabular_inspect_options_from_arguments(arguments): + """Inspection options from plan arguments.""" + arguments = arguments if isinstance(arguments, Mapping) else {} + sample_rows = arguments.get("sample_rows") + return TabularInspectOptions( + sheet=arguments.get("sheet") or None, + sheets=arguments.get("sheets") or SHEETS_FIRST, + header_row=arguments.get("header_row"), + sample_rows=DEFAULT_INSPECT_SAMPLE_ROWS if sample_rows is None else sample_rows, + ) + + +# --------------------------------------------------------------------------- +# Prepared column mappings +# --------------------------------------------------------------------------- + +TABULAR_COLUMN_MAPPING_PROFILE = "tabular_column_mapping_v1" +MAPPING_CONFIDENCE_LEVELS = ("high", "medium", "low") +MAX_MAPPING_ENTRIES = 1024 +MAX_MAPPING_NOTES_CHARS = 2000 + + +def tabular_column_mapping_schema(): + """The prepared-content profile a compose step fills to line up differently named columns.""" + name = {"type": "string", "minLength": 1, "maxLength": MAX_COLUMN_NAME_BYTES} + return { + "type": "object", + "description": ( + "How to line up the columns of several CSV or Excel files before merging them. columns " + "is the merged table's columns in order; use the clearest existing header for each. " + "mappings lists every header whose name differs from the column it fills, and every " + "header to leave out (target_column null). A header that already matches a column, " + "ignoring case and spacing, needs no entry. Map a header only when it holds the same " + "kind of value; when unsure, keep it as its own column or mark confidence low." + ), + "properties": { + "columns": { + "type": "array", "items": dict(name), "minItems": 1, "maxItems": MAX_MAPPED_COLUMNS, + "uniqueItems": True, + }, + "mappings": { + "type": "array", + "maxItems": MAX_MAPPING_ENTRIES, + "items": { + "type": "object", + "properties": { + "source_column": dict(name, description="A header exactly as it appears in a file."), + "target_column": { + "type": ["string", "null"], "maxLength": MAX_COLUMN_NAME_BYTES, + "description": "The column in columns this header fills, or null to leave it out.", + }, + "confidence": {"type": "string", "enum": list(MAPPING_CONFIDENCE_LEVELS)}, + }, + "required": ["source_column", "target_column", "confidence"], + "additionalProperties": False, + }, + }, + "notes": { + "type": "string", "maxLength": MAX_MAPPING_NOTES_CHARS, + "description": "A short explanation of the mapping for the user.", + }, + }, + "required": ["columns", "mappings"], + "additionalProperties": False, + } + + +def tabular_mapping_from_profile(value): + """Validate a prepared ``tabular_column_mapping_v1`` value; return merge columns and aliases. + + Raises ``invalid_mapping`` when the value breaks the profile: unknown fields, repeated + columns or headers, a target that isn't a listed column, or a header that is itself a + different listed column. + """ + def invalid(message): + return TabularMergeError("invalid_mapping", message) + + if not isinstance(value, Mapping) or set(value) - {"columns", "mappings", "notes"}: + raise invalid("A column mapping has only columns, mappings and notes.") + raw_columns = value.get("columns") + raw_mappings = value.get("mappings") + notes = value.get("notes", "") + if not isinstance(raw_columns, list) or not 1 <= len(raw_columns) <= MAX_MAPPED_COLUMNS: + raise invalid(f"A column mapping lists 1 to {MAX_MAPPED_COLUMNS} columns.") + if not isinstance(raw_mappings, list) or len(raw_mappings) > MAX_MAPPING_ENTRIES: + raise invalid(f"A column mapping has at most {MAX_MAPPING_ENTRIES} mappings.") + if not isinstance(notes, str) or len(notes) > MAX_MAPPING_NOTES_CHARS: + raise invalid(f"Mapping notes have at most {MAX_MAPPING_NOTES_CHARS} characters.") + try: + columns = _column_names(raw_columns, "The mapped columns", maximum=MAX_MAPPED_COLUMNS) + except TabularMergeError as exc: + raise invalid(exc.message) from exc + targets = {column.casefold(): column for column in columns} + aliases = {} + ignored = [] + low_confidence = [] + sources = set() + for item in raw_mappings: + if not isinstance(item, Mapping) or set(item) != {"source_column", "target_column", "confidence"}: + raise invalid("Each mapping has exactly source_column, target_column and confidence.") + if item["confidence"] not in MAPPING_CONFIDENCE_LEVELS: + raise invalid(f"Mapping confidence is one of: {', '.join(MAPPING_CONFIDENCE_LEVELS)}.") + try: + source = _column_name(item["source_column"], "Mapped headers") + except TabularMergeError as exc: + raise invalid(exc.message) from exc + key = source.casefold() + if key in sources: + raise invalid(f'"{source}" is mapped more than once.') + sources.add(key) + target = item["target_column"] + if target is None: + if key in targets: + raise invalid(f'"{source}" is a listed column, so it can\'t be left out.') + ignored.append(source) + continue + target_key = _display_header(target).casefold() if isinstance(target, str) else "" + if target_key not in targets: + raise invalid(f'"{source}" is mapped to a column that isn\'t listed.') + if key == target_key: + continue + if key in targets: + raise invalid(f'"{source}" is a listed column, so it can\'t also fill "{targets[target_key]}".') + aliases.setdefault(targets[target_key], []).append(source) + if item["confidence"] == "low": + low_confidence.append({"source_column": source, "target_column": targets[target_key]}) + for target, names in aliases.items(): + if len(names) > MAX_ALIASES_PER_TARGET: + raise invalid(f'At most {MAX_ALIASES_PER_TARGET} headers can fill "{target}".') + return { + "columns": columns, + "column_aliases": aliases, + "ignored": ignored, + "low_confidence": low_confidence, + "notes": notes.strip(), + } + + def iter_csv_rows(content, file_name="A selected file"): """Decode CSV bytes; return ``(encoding, delimiter, rows)`` where rows yields lists of text.""" source = TabularMergeSource(file_name, file_name, lambda: content) @@ -188,37 +656,56 @@ def normalize_header_name(value): return _display_header(value).casefold() +def squash_header_name(value): + """A looser key that also ignores spaces, punctuation and underscores, for near matches.""" + return _NOT_WORD.sub("", normalize_header_name(value)) + + def _display_header(value): text = unicodedata.normalize("NFC", str(value if value is not None else "")) return " ".join(text.split()) +# --------------------------------------------------------------------------- +# Spooled rows and the merge result +# --------------------------------------------------------------------------- + + class _RowSpool: """Validated rows as compact JSON lines; small merges stay in memory.""" def __init__(self): self._file = tempfile.SpooledTemporaryFile(max_size=_SPOOL_MEMORY_BYTES, mode="w+b") + self._offsets = array("q") + self._end = 0 self.count = 0 self.closed = False def append(self, values): - self._file.write(json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) - self._file.write(b"\n") + data = json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + b"\n" + self._file.seek(self._end) + self._file.write(data) + self._offsets.append(self._end) + self._end += len(data) self.count += 1 - def iter_rows(self): + def _read(self, index): + self._file.seek(self._offsets[index]) + line = self._file.readline() + if not line: + raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") + return json.loads(line) + + def iter_indexed(self, order=None): + """``(index, values)`` in spool order, or in ``order`` when given.""" if self.closed: raise TabularMergeError("result_closed", "The merged rows are no longer available.") self._file.flush() - self._file.seek(0) - yielded = 0 - for line in self._file: - if not line.strip(): - continue - yielded += 1 - yield json.loads(line) - if yielded != self.count: - raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") + indices = range(self.count) if order is None else order + for index in indices: + if self.closed: + raise TabularMergeError("result_closed", "The merged rows are no longer available.") + yield index, self._read(index) def close(self): if not self.closed: @@ -233,29 +720,48 @@ class TabularMergeResult: count. Use it as a context manager so the spooled rows are always released. """ - def __init__(self, columns, report, spool): + def __init__(self, columns, report, spool, *, nullable=(), row_count=None, order=None, keep=None): self._columns = tuple(columns) + self._nullable = frozenset(nullable) self._report = report self._spool = spool + self._row_count = spool.count if row_count is None else row_count + self._order = order + self._keep = keep @property def columns(self): return self._columns + @property + def nullable_columns(self): + """Columns that hold null for rows from files that don't have them.""" + return self._nullable + @property def row_count(self): - return self._spool.count + return self._row_count @property def report(self): return json.loads(json.dumps(self._report)) def iter_rows(self): - return self._spool.iter_rows() + width = len(self._columns) + yielded = 0 + for index, values in self._spool.iter_indexed(self._order): + if len(values) < width: + values.extend([None] * (width - len(values))) + if self._keep is not None and not self._keep(index, values): + continue + yielded += 1 + yield values + if yielded != self._row_count: + raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") def iter_records(self): columns = self._columns - for values in self._spool.iter_rows(): + for values in self.iter_rows(): yield dict(zip(columns, values)) def close(self): @@ -269,6 +775,11 @@ def __exit__(self, exc_type, exc, traceback): return False +# --------------------------------------------------------------------------- +# Merging +# --------------------------------------------------------------------------- + + def merge_tabular_sources( sources: Sequence[TabularMergeSource], *, @@ -277,110 +788,454 @@ def merge_tabular_sources( cancel_requested: Optional[Callable[[], bool]] = None, on_progress: Optional[Callable[[dict], None]] = None, ) -> TabularMergeResult: - """Append the rows of same-structure sources, in the given order, into one table. + """Append the rows of the sources, in the given order, into one table. - The first source defines the column order. ``by_name`` accepts the same columns in - any order after header normalization; ``exact_order`` also requires the same order. - A schema mismatch reads every remaining header so the report lists all mismatches. + The first file (or sheet) defines the column order for ``by_name`` and ``exact_order``; + ``union`` adds each new column the first time a file has it; ``mapped`` uses exactly + the requested columns. Under ``on_incompatible="fail"`` a mismatch reads every remaining + header so the report lists all mismatches, then nothing is merged. """ options = options or TabularMergeOptions() limits = limits or TabularMergeLimits() sources = list(sources or ()) _validate_sources(sources, limits) - - report = _new_report(options, len(sources)) - spool = _RowSpool() - reference = None - output_columns = () - source_column = None - mismatch_found = False - blank_rows_total = 0 + merger = _Merger(options, limits, len(sources), cancel_requested) try: for index, source in enumerate(sources): _check_cancel(cancel_requested) - entry = _new_source_entry(source) - report["sources"].append(entry) - content = _load_source_bytes(source, limits) - table = _open_table(source, content, options, limits) + rows = merger.add_source(source) + if on_progress is not None: + on_progress({ + "index": index + 1, "total": len(sources), "file_name": source.file_name, "rows": rows, + }) + return merger.finish() + except BaseException: + merger.discard() + raise + + +class _Merger: + """The single-pass state of one merge.""" + + def __init__(self, options, limits, source_count, cancel_requested): + self.options = options + self.limits = limits + self.cancel_requested = cancel_requested + self.report = _new_report(options, source_count) + self.alias_index = _alias_index(options) + self.provenance = None + self.data_display = [] + self.data_keys = [] + self.data_index = {} + self.seen_counts = [] + self.reference = None + self.mismatch = False + self.merged_tables = 0 + self.excluded_tables = 0 + self.rows_read = 0 + self.blank_rows = 0 + self.duplicates_removed = 0 + self.sheet_null = False + self.dedupe_keys = [normalize_header_name(name) for name in options.dedupe_columns] + self.sort_keys = [normalize_header_name(item.column) for item in options.sort_by] + self.seen_digests = set() + self.last_digest = {} + self.row_starts = [] + self.row_entries = [] + self.sort_values = [] + if options.schema_policy == SCHEMA_POLICY_MAPPED: + for name in options.columns: + self._add_data_column(name, normalize_header_name(name)) + self._init_provenance(set(self.data_keys)) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + # Created last, so a refused configuration leaves nothing to release. + self.spool = _RowSpool() + + def discard(self): + self.spool.close() + + def add_source(self, source): + content = _load_source_bytes(source, self.limits) + reader = _open_reader(source, content, self.limits) + rows = 0 + try: try: - entry.update(table.describe()) - header = _header_names(table.header, source, limits, entry) - if reference is None: - reference = header - source_column = _source_column_name(options, header, report) - output_columns = ((source_column,) if source_column else ()) + tuple( - display for display, _ in header - ) - if len(output_columns) > limits.max_columns: - raise TabularMergeError( - "too_many_columns", - f"The merged table would have {len(output_columns)} columns; " - f"at most {limits.max_columns} are supported.", - ) - report["columns"] = list(output_columns) - report["source_column"] = source_column - column_map = list(range(len(header))) - else: - column_map = _match_header(reference, header, options.schema_policy, entry) - if column_map is None: - mismatch_found = True - entry["status"] = SOURCE_STATUS_SCHEMA_MISMATCH + plan = reader.plan(self.options, self.limits) + except TabularMergeError as exc: + if exc.code != "sheet_not_found" or self.options.on_incompatible != ON_INCOMPATIBLE_EXCLUDE: + raise + entry = _new_source_entry(source, reader, self.options.sheet) + self.report["sources"].append(entry) + self._incompatible(entry, "sheet_not_found") + return 0 + for sheet, skip_reason in plan: + _check_cancel(self.cancel_requested) + entry = _new_source_entry(source, reader, sheet) + self.report["sources"].append(entry) + if skip_reason is not None: + entry["status"] = SOURCE_STATUS_SKIPPED + entry["reason"] = skip_reason + continue + try: + table = reader.open_table(sheet, self.options.header_row) + except TabularMergeError as exc: + if exc.code != "empty_source" or isinstance(exc, TabularMergeCancelled): + raise + if self.options.sheets == SHEETS_ALL: + entry["status"] = SOURCE_STATUS_SKIPPED + entry["reason"] = "no_header_row" continue - - if mismatch_found: - entry["status"] = SOURCE_STATUS_NOT_PROCESSED + if self.options.on_incompatible != ON_INCOMPATIBLE_EXCLUDE: + raise + self._incompatible(entry, "no_header_row") continue + try: + entry["header_row"] = table.header_row + self._add_table(source, table, entry) + finally: + table.close() + rows += entry["rows"] + finally: + reader.close() + return rows + + # -- columns ----------------------------------------------------------- + + def _add_data_column(self, display, key): + self.data_index[key] = len(self.data_keys) + self.data_keys.append(key) + self.data_display.append(display) + self.seen_counts.append(0) + + def _init_provenance(self, taken_keys): + names = [] + if self.options.include_source_column: + taken = set(taken_keys) + names.append(self._claim_provenance_name( + _display_header(self.options.source_column_name), taken, "source_column_renamed", + "file name column", + )) + if self.options.sheets == SHEETS_ALL: + taken.add(names[0].casefold()) + names.append(self._claim_provenance_name( + DEFAULT_SHEET_COLUMN_NAME, taken, "sheet_column_renamed", "sheet name column", + )) + self.provenance = names + self.report["source_column"] = names[0] if names else None + self.report["sheet_column"] = names[1] if len(names) > 1 else None + self._check_width() + + def _claim_provenance_name(self, base, taken, code, label): + candidate = base + suffix = 2 + while candidate.casefold() in taken: + candidate = _suffixed_name(base, suffix) + suffix += 1 + if candidate != base: + self.report["warnings"].append({ + "code": code, + "message": f'The files already have a "{base}" column, so the {label} is "{candidate}".', + }) + return candidate + + def _unique_data_name(self, display): + taken = {name.casefold() for name in (self.provenance or []) + self.data_display} + if display.casefold() not in taken: + return display + suffix = 2 + candidate = _suffixed_name(display, suffix) + while candidate.casefold() in taken: + suffix += 1 + candidate = _suffixed_name(display, suffix) + self.report["warnings"].append({ + "code": "column_renamed", + "message": f'A column named "{display}" is shown as "{candidate}" because that name is already used.', + }) + return candidate + + def _check_width(self): + width = len(self.provenance or ()) + len(self.data_keys) + if width > self.limits.max_columns: + raise TabularMergeError( + "too_many_columns", + f"The merged table would have {width} columns; at most {self.limits.max_columns} are supported.", + ) - rows, blank_rows = _append_rows( - table, source, column_map, len(header), source_column, spool, limits, - cancel_requested, entry, + def _require_named_columns(self, keys, code, purpose): + missing = [key for key in keys if key not in self.data_index] + if missing: + raise TabularMergeError( + code, f"The column(s) to {purpose} by aren't in the merged table: {', '.join(_quoted(missing))}.", + ) + + # -- tables ------------------------------------------------------------ + + def _add_table(self, source, table, entry): + header = _header_names(table.header, source, self.limits, entry) + header = _apply_aliases(header, self.alias_index, source, entry) + policy = self.options.schema_policy + if policy in (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER): + if self.reference is None: + self.reference = header + for display, key in header: + self._add_data_column(display, key) + self._init_provenance(set(self.data_keys)) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + mapping = list(range(len(header))) + else: + mapping = _match_header(self.reference, header, policy, entry) + if mapping is None: + self._incompatible(entry, "columns_differ") + return + elif policy == SCHEMA_POLICY_UNION: + mapping = self._union_mapping(header, entry) + else: + mapping = self._mapped_mapping(header, entry) + if mapping is None: + self._incompatible(entry, "no_matching_columns") + return + + if self.mismatch: + entry["status"] = SOURCE_STATUS_NOT_PROCESSED + return + self._append_rows(source, table, entry, mapping, len(header)) + entry["status"] = SOURCE_STATUS_MERGED + self.merged_tables += 1 + for position, column in enumerate(mapping): + if column is not None: + self.seen_counts[position] += 1 + + def _union_mapping(self, header, entry): + first = self.provenance is None + if first: + self._init_provenance({key for _, key in header}) + added = [] + for display, key in header: + if key not in self.data_index: + name = self._unique_data_name(display) + self._add_data_column(name, key) + self._check_width() + added.append(name) + positions = {key: index for index, (_, key) in enumerate(header)} + mapping = [positions.get(key) for key in self.data_keys] + if not first: + entry["missing_columns"] = [ + name for name, column in zip(self.data_display, mapping) if column is None + ] + entry["extra_columns"] = added + return mapping + + def _mapped_mapping(self, header, entry): + positions = {key: index for index, (_, key) in enumerate(header)} + mapping = [positions.get(key) for key in self.data_keys] + entry["missing_columns"] = [name for name, column in zip(self.data_display, mapping) if column is None] + entry["ignored_columns"] = [display for display, key in header if key not in self.data_index] + if all(column is None for column in mapping): + return None + return mapping + + def _incompatible(self, entry, reason): + entry["reason"] = reason + if self.options.on_incompatible == ON_INCOMPATIBLE_EXCLUDE: + entry["status"] = SOURCE_STATUS_EXCLUDED + self.excluded_tables += 1 + else: + entry["status"] = SOURCE_STATUS_SCHEMA_MISMATCH + self.mismatch = True + + # -- rows -------------------------------------------------------------- + + def _append_rows(self, source, table, entry, mapping, width): + options = self.options + limits = self.limits + prefix = [] + if self.provenance: + prefix.append(source.file_name) + if len(self.provenance) > 1: + prefix.append(table.sheet) + if table.sheet is None: + self.sheet_null = True + self.row_starts.append(self.spool.count) + self.row_entries.append(entry) + dedupe_positions = ( + [self.data_index.get(key) for key in self.dedupe_keys] + if options.dedupe == DEDUPE_KEY_COLUMNS else None + ) + sort_positions = [self.data_index.get(key) for key in self.sort_keys] + keep_last = options.dedupe != DEDUPE_NONE and options.dedupe_keep == DEDUPE_KEEP_LAST + checked = 0 + for row_number, values in table.rows(): + if checked % _CANCEL_CHECK_INTERVAL == 0: + _check_cancel(self.cancel_requested) + checked += 1 + cells = list(values) + if len(cells) > width: + if any(_cell_has_content(value) for value in cells[width:]): + raise TabularMergeError( + "row_has_extra_values", + f"Row {row_number} in {_safe_name(source)} has more values than there are columns.", + ) + cells = cells[:width] + if not any(_cell_has_content(value) for value in cells): + entry["blank_rows_skipped"] += 1 + self.blank_rows += 1 + continue + if len(cells) < width: + entry["short_rows_padded"] += 1 + cells.extend([""] * (width - len(cells))) + for value in cells: + if len(value) > limits.max_cell_chars: + raise TabularMergeError( + "cell_too_large", + f"Row {row_number} in {_safe_name(source)} has a value longer than " + f"{limits.max_cell_chars} characters.", + ) + if self.rows_read >= limits.max_total_rows: + raise TabularMergeError( + "row_limit_exceeded", + f"The selected files hold more than {limits.max_total_rows:,} rows. " + "Merge fewer files here, or use a workflow for larger merges.", ) - blank_rows_total += blank_rows - entry["rows"] = rows - entry["blank_rows_skipped"] = blank_rows - entry["status"] = SOURCE_STATUS_MERGED - finally: - table.close() - if on_progress is not None: - on_progress({ - "index": index + 1, "total": len(sources), - "file_name": source.file_name, "rows": entry.get("rows", 0), - }) + self.rows_read += 1 + data = [cells[column] if column is not None else None for column in mapping] + if options.dedupe != DEDUPE_NONE: + digest = _row_digest( + data if dedupe_positions is None else _values_at(data, dedupe_positions) + ) + if digest is not None: + if keep_last: + previous = self.last_digest.get(digest) + if previous is not None: + owner = self._entry_for_row(previous) + owner["rows"] -= 1 + owner["duplicates_removed"] += 1 + self.duplicates_removed += 1 + self.last_digest[digest] = self.spool.count + elif digest in self.seen_digests: + entry["duplicates_removed"] += 1 + self.duplicates_removed += 1 + continue + else: + self.seen_digests.add(digest) + if self.sort_keys: + if self.spool.count >= limits.max_sort_rows: + raise TabularMergeError( + "sort_limit_exceeded", + f"Sorting is available for merges of up to {limits.max_sort_rows:,} rows.", + ) + self.sort_values.append(tuple( + _sort_value(data[column] if column is not None else None, item) + for column, item in zip(sort_positions, options.sort_by) + )) + self.spool.append(prefix + data) + entry["rows"] += 1 + + def _entry_for_row(self, index): + return self.row_entries[bisect.bisect_right(self.row_starts, index) - 1] + + # -- finish ------------------------------------------------------------ - if mismatch_found: + def finish(self): + report = self.report + if self.mismatch: report["status"] = "failed" + report["columns"] = list(self.provenance or ()) + list(self.data_display) for entry in report["sources"]: if entry["status"] in (SOURCE_STATUS_MERGED, SOURCE_STATUS_NOT_PROCESSED): entry["status"] = SOURCE_STATUS_COMPATIBLE entry["rows"] = 0 entry["blank_rows_skipped"] = 0 entry["short_rows_padded"] = 0 + entry["duplicates_removed"] = 0 + raise TabularMergeError("schema_mismatch", _mismatch_message(report), report=report) + if self.merged_tables == 0: + report["status"] = "failed" raise TabularMergeError( - "schema_mismatch", _mismatch_message(report), report=report, + "nothing_to_merge", + "None of the selected files has a sheet with column headers and matching columns to merge.", + report=report, ) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + + columns = tuple(self.provenance) + tuple(self.data_display) + nullable = set() + if len(self.provenance) > 1 and self.sheet_null: + nullable.add(self.provenance[1]) + for name, seen in zip(self.data_display, self.seen_counts): + if seen < self.merged_tables: + nullable.add(name) + keep_last = self.options.dedupe != DEDUPE_NONE and self.options.dedupe_keep == DEDUPE_KEEP_LAST + row_count = self.spool.count - (self.duplicates_removed if keep_last else 0) + order = self._sort_order() if self.sort_keys else None + keep = self._keep_last_predicate() if keep_last else None report["status"] = "merged" + report["columns"] = list(columns) + report["nullable_columns"] = [name for name in columns if name in nullable] report["totals"] = { - "sources": len(sources), - "rows": spool.count, - "columns": len(output_columns), - "blank_rows_skipped": blank_rows_total, + "sources": report["totals"]["sources"], + "tables_merged": self.merged_tables, + "rows": row_count, + "columns": len(columns), + "blank_rows_skipped": self.blank_rows, + "duplicates_removed": self.duplicates_removed, + "excluded": self.excluded_tables, } + limitations = [] if any(entry.get("format") != TABULAR_MERGE_FORMAT_CSV for entry in report["sources"]): - report["limitations"] = list(_EXCEL_LIMITATIONS) - return TabularMergeResult(output_columns, report, spool) - except BaseException: - spool.close() - raise + limitations.extend(_EXCEL_LIMITATIONS) + if self.excluded_tables: + considered = sum( + entry["status"] in (SOURCE_STATUS_MERGED, SOURCE_STATUS_EXCLUDED) for entry in report["sources"] + ) + limitations.append(_EXCLUDED_LIMITATION.format(count=self.excluded_tables, total=considered)) + if any(name in nullable for name in self.data_display): + limitations.append(_BLANK_COLUMNS_LIMITATION) + if any(entry.get("ignored_columns") for entry in report["sources"] if entry["status"] == SOURCE_STATUS_MERGED): + limitations.append(_IGNORED_COLUMNS_LIMITATION) + report["limitations"] = limitations + result = TabularMergeResult( + columns, report, self.spool, nullable=nullable, row_count=row_count, order=order, keep=keep, + ) + self.sort_values = [] + return result + + def _sort_order(self): + values = self.sort_values + order = list(range(self.spool.count)) + for position in reversed(range(len(self.options.sort_by))): + descending = self.options.sort_by[position].descending + order.sort(key=lambda index, position=position: values[index][position], reverse=descending) + return order + + def _keep_last_predicate(self): + offset = len(self.provenance) + positions = ( + [self.data_index[key] for key in self.dedupe_keys] + if self.options.dedupe == DEDUPE_KEY_COLUMNS else None + ) + last = self.last_digest + + def keep(index, values): + data = values[offset:] + digest = _row_digest(data if positions is None else _values_at(data, positions)) + return digest is None or last.get(digest) == index + + return keep -def _validate_sources(sources, limits): - if len(sources) < 2: - raise TabularMergeError("too_few_sources", "Merging needs at least two files.") +def _validate_sources(sources, limits, *, minimum=2): + if len(sources) < minimum: + raise TabularMergeError( + "too_few_sources", "Merging needs at least two files." if minimum > 1 else "Select a file to inspect.", + ) if len(sources) > limits.max_sources: raise TabularMergeError( "too_many_sources", - f"{len(sources)} files were selected; at most {limits.max_sources} can be merged here.", + f"{len(sources)} files were selected; at most {limits.max_sources} can be used here.", ) seen = set() for source in sources: @@ -404,38 +1259,45 @@ def _new_report(options, source_count): return { "version": TABULAR_MERGE_REPORT_VERSION, "status": "running", - "policy": { - "schema_policy": options.schema_policy, - "sheet": options.sheet, - "include_source_column": options.include_source_column, - "source_column_name": _display_header(options.source_column_name) if options.include_source_column else None, - }, + "policy": options.policy_report(), "columns": [], + "nullable_columns": [], "source_column": None, - "totals": {"sources": source_count, "rows": 0, "columns": 0, "blank_rows_skipped": 0}, + "sheet_column": None, + "totals": { + "sources": source_count, "tables_merged": 0, "rows": 0, "columns": 0, + "blank_rows_skipped": 0, "duplicates_removed": 0, "excluded": 0, + }, "sources": [], "warnings": [], "limitations": [], } -def _new_source_entry(source): - return { +def _new_source_entry(source, reader, sheet): + entry = { "source_id": source.source_id, "file_name": _safe_name(source), "format": TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()), - "sheet": None, + "sheet": sheet, "encoding": None, "delimiter": None, + "header_row": None, "rows": 0, "blank_rows_skipped": 0, "short_rows_padded": 0, + "duplicates_removed": 0, "status": SOURCE_STATUS_NOT_PROCESSED, + "reason": None, "missing_columns": [], "extra_columns": [], + "ignored_columns": [], + "renamed_columns": [], "order_differs": False, "warnings": [], } + entry.update(reader.describe()) + return entry def _safe_name(source): @@ -443,6 +1305,14 @@ def _safe_name(source): return name or "A selected file" +def _suffixed_name(base, suffix): + ending = f" ({suffix})" + name = base + while len((name + ending).encode("utf-8")) > MAX_COLUMN_NAME_BYTES and name: + name = name[:-1].rstrip() + return name + ending + + def _check_cancel(cancel_requested): if cancel_requested is not None and cancel_requested(): raise TabularMergeCancelled() @@ -462,22 +1332,13 @@ def _load_source_bytes(source, limits): return content -def _source_column_name(options, header, report): - if not options.include_source_column: - return None - base = _display_header(options.source_column_name) - taken = {normalized for _, normalized in header} - candidate = base - suffix = 2 - while candidate.casefold() in taken: - candidate = f"{base} ({suffix})" - suffix += 1 - if candidate != base: - report["warnings"].append({ - "code": "source_column_renamed", - "message": f'The files already have a "{base}" column, so the file name column is "{candidate}".', - }) - return candidate +def _alias_index(options): + index = {} + for target, aliases in options.column_aliases: + target_key = normalize_header_name(target) + for alias in aliases: + index[normalize_header_name(alias)] = (target, target_key) + return index def _header_names(raw_header, source, limits, entry): @@ -513,6 +1374,29 @@ def _header_names(raw_header, source, limits, entry): return names +def _apply_aliases(header, alias_index, source, entry): + """Rename headers that are aliases of a target column; renaming must not duplicate one.""" + if not alias_index: + return header + renamed = [] + result = [] + seen = set() + for display, key in header: + target = alias_index.get(key) + if target is not None: + renamed.append({"from": display, "to": target[0]}) + display, key = target + if key in seen: + raise TabularMergeError( + "duplicate_columns", + f'{_safe_name(source)} has more than one column that becomes "{display}".', + ) + seen.add(key) + result.append((display, key)) + entry["renamed_columns"] = renamed + return result + + def _match_header(reference, header, schema_policy, entry): """Positions in ``header`` for each reference column, or None on a mismatch.""" reference_keys = [normalized for _, normalized in reference] @@ -539,13 +1423,16 @@ def _mismatch_message(report): if entry["status"] != SOURCE_STATUS_SCHEMA_MISMATCH: continue details = [] - if entry["missing_columns"]: + if entry["reason"] == "no_matching_columns": + details.append("none of the requested columns") + if entry["missing_columns"] and entry["reason"] != "no_matching_columns": details.append(f"missing {', '.join(_quoted(entry['missing_columns']))}") if entry["extra_columns"]: details.append(f"extra {', '.join(_quoted(entry['extra_columns']))}") if not details and entry["order_differs"]: details.append("columns in a different order") - parts.append(f"{entry['file_name']} ({'; '.join(details)})") + label = entry["file_name"] if not entry.get("sheet") else f'{entry["file_name"]} sheet "{entry["sheet"]}"' + parts.append(f"{label} ({'; '.join(details)})") return ( f"These files don't have the same columns as {reference}: {'; '.join(parts)}. " "Nothing was merged." @@ -553,57 +1440,312 @@ def _mismatch_message(report): def _quoted(values, limit=8): + values = list(values) shown = [f'"{value}"' for value in values[:limit]] if len(values) > limit: shown.append(f"and {len(values) - limit} more") return shown -def _append_rows(table, source, column_map, width, source_column, spool, limits, cancel_requested, entry): - rows = 0 - blank_rows = 0 +def _cell_has_content(value): + return bool(value) and bool(str(value).strip()) + + +def _values_at(data, positions): + return [data[position] if position is not None and position < len(data) else None for position in positions] + + +def _row_digest(values): + """A collision-resistant key for duplicate detection, or None when every value is blank. + + Missing (null) and empty values compare equal, and trailing blanks are ignored, so a row + read before a later file added a column matches the same row read after it. + """ + normalized = ["" if value is None else value for value in values] + while normalized and normalized[-1] == "": + normalized.pop() + if not any(_cell_has_content(value) for value in normalized): + return None + encoded = json.dumps(normalized, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return hashlib.blake2b(encoded, digest_size=16).digest() + + +def _sort_value(value, sort): + """A comparable key; typed values sort before text that isn't that type, blanks last.""" + ascending = not sort.descending + typed_rank = 0 if ascending else 2 + blank_rank = 2 if ascending else 0 + if value is None or not str(value).strip(): + return (blank_rank, "") + if sort.value_type == SORT_TYPE_NUMBER: + number = _parse_number(value) + if number is not None: + return (typed_rank, number) + return (1, value.casefold()) + if sort.value_type == SORT_TYPE_DATE: + moment = _parse_date(value) + if moment is not None: + return (typed_rank, moment) + return (1, value.casefold()) + return (typed_rank, value.casefold()) + + +def _parse_number(value): + text = value.strip() + if text[:1] in _CURRENCY_PREFIXES: + text = text[1:].strip() + if not _NUMBER_TEXT.fullmatch(text): + return None + try: + number = float(text.replace(",", "")) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _parse_date(value): + text = value.strip() + if len(text) < 10 or not text[:4].isdigit(): + return None + try: + if len(text) == 10: + day = date.fromisoformat(text) + return datetime(day.year, day.month, day.day) + moment = datetime.fromisoformat(text) + except ValueError: + return None + if moment.tzinfo is not None: + moment = moment.astimezone(timezone.utc).replace(tzinfo=None) + return moment + + +# --------------------------------------------------------------------------- +# Inspection +# --------------------------------------------------------------------------- + + +def inspect_tabular_sources( + sources: Sequence[TabularMergeSource], + *, + options: Optional[TabularInspectOptions] = None, + limits: Optional[TabularMergeLimits] = None, + cancel_requested: Optional[Callable[[], bool]] = None, +) -> dict: + """Describe each file's sheets, headers, row counts and samples, and how their columns line up. + + A file that can't be read is reported with its problem rather than failing the inspection. + """ + options = options or TabularInspectOptions() + limits = limits or TabularMergeLimits() + sources = list(sources or ()) + _validate_sources(sources, limits, minimum=1) + report = { + "version": TABULAR_INSPECTION_VERSION, + "options": { + "sheet": options.sheet, "sheets": options.sheets, + "header_row": options.header_row, "sample_rows": options.sample_rows, + }, + "sources": [], + "compatibility": None, + "samples_omitted": False, + "limitations": [], + } + inspected = [] + for source in sources: + _check_cancel(cancel_requested) + entry = { + "source_id": source.source_id, + "file_name": _safe_name(source), + "format": TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()), + "status": "inspected", + "problem": None, + "encoding": None, + "delimiter": None, + "sheets": [], + "tables": [], + } + report["sources"].append(entry) + reader = None + try: + content = _load_source_bytes(source, limits) + reader = _open_reader(source, content, limits) + entry.update(reader.describe()) + entry["sheets"] = [{"name": name, "visible": state == "visible"} for name, state in reader.sheet_states()] + for sheet, skip_reason in reader.plan(options, limits): + _check_cancel(cancel_requested) + table_entry = {"sheet": sheet, "status": "inspected", "reason": None, "problem": None} + entry["tables"].append(table_entry) + if skip_reason is not None: + table_entry.update(status=SOURCE_STATUS_SKIPPED, reason=skip_reason) + continue + try: + table = reader.open_table(sheet, options.header_row) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + table_entry.update(status="problem", problem={"code": exc.code, "message": exc.message}) + continue + try: + header = _inspect_table(source, table, table_entry, options, limits, cancel_requested) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + table_entry.update(status="problem", problem={"code": exc.code, "message": exc.message}) + continue + finally: + table.close() + inspected.append((entry, table_entry, header)) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + entry["status"] = "problem" + entry["problem"] = {"code": exc.code, "message": exc.message} + finally: + if reader is not None: + reader.close() + + report["compatibility"] = _compatibility(inspected) + limitations = ["Row counts leave out blank rows."] + if options.sample_rows: + limitations.append( + f"Samples show up to {options.sample_rows} row(s) per sheet, and sample values longer than " + f"{_INSPECT_CELL_CHARS} characters are shortened." + ) + if any(entry["format"] != TABULAR_MERGE_FORMAT_CSV for entry in report["sources"]): + limitations.append("Excel formulas are shown as their last calculated values.") + report["limitations"] = limitations + _fit_inspection_budget(report) + return report + + +def _inspect_table(source, table, table_entry, options, limits, cancel_requested): + probe = {"warnings": []} + header = _header_names(table.header, source, limits, probe) + width = len(header) + samples = [] + count = 0 + capped = False + extra_rows = 0 + first_data_row = None for row_number, values in table.rows(): - if rows % _CANCEL_CHECK_INTERVAL == 0: + if count % _CANCEL_CHECK_INTERVAL == 0: _check_cancel(cancel_requested) cells = list(values) - if len(cells) > width: - if any(_cell_has_content(value) for value in cells[width:]): - raise TabularMergeError( - "row_has_extra_values", - f"Row {row_number} in {_safe_name(source)} has more values than there are columns.", - ) - cells = cells[:width] if not any(_cell_has_content(value) for value in cells): - blank_rows += 1 continue - if len(cells) < width: - entry["short_rows_padded"] += 1 - cells.extend([""] * (width - len(cells))) - for value in cells: - if len(value) > limits.max_cell_chars: - raise TabularMergeError( - "cell_too_large", - f"Row {row_number} in {_safe_name(source)} has a value longer than " - f"{limits.max_cell_chars} characters.", - ) - if spool.count >= limits.max_total_rows: - raise TabularMergeError( - "row_limit_exceeded", - f"The merged table would have more than {limits.max_total_rows:,} rows. " - "Merge fewer files here, or use a workflow for larger merges.", - ) - ordered = [cells[position] for position in column_map] - spool.append(([source.file_name] if source_column else []) + ordered) - rows += 1 - return rows, blank_rows + if first_data_row is None: + first_data_row = (row_number, cells) + if len(cells) > width and any(_cell_has_content(value) for value in cells[width:]): + extra_rows += 1 + count += 1 + if len(samples) < options.sample_rows: + row = [_clip_sample(value) for value in cells[:width]] + samples.append(row + [""] * (width - len(row))) + if count >= limits.max_total_rows: + capped = True + break + warnings = list(probe["warnings"]) + header_values = sum(_cell_has_content(value) for value in table.header) + if ( + options.header_row is None and header_values == 1 and first_data_row is not None + and sum(_cell_has_content(value) for value in first_data_row[1]) > 1 + ): + warnings.append({ + "code": "possible_title_row", + "message": ( + f"Row {table.header_row} has only one value, so it may be a title rather than column " + f"headers; row {first_data_row[0]} may hold the headers." + ), + "suggested_header_row": first_data_row[0], + }) + if extra_rows: + warnings.append({ + "code": "rows_with_extra_values", + "message": f"{extra_rows} row(s) have more values than there are column headers.", + }) + table_entry.update({ + "header_row": table.header_row, + "columns": [display for display, _ in header], + "row_count": count, + "row_count_capped": capped, + "sample_rows": samples, + "warnings": warnings, + }) + return header + + +def _clip_sample(value): + text = "" if value is None else str(value) + if len(text) > _INSPECT_CELL_CHARS: + return text[:_INSPECT_CELL_CHARS - 1] + "\u2026" + return text + + +def _compatibility(inspected): + groups = {} + order = None + same_order = True + presence = {} + for entry, table_entry, header in inspected: + keys = [key for _, key in header] + group = groups.setdefault( + tuple(sorted(keys)), {"columns": [display for display, _ in header], "tables": []}, + ) + group["tables"].append({ + "source_id": entry["source_id"], "file_name": entry["file_name"], "sheet": table_entry["sheet"], + }) + if order is None: + order = keys + elif keys != order: + same_order = False + for display, key in header: + item = presence.setdefault(key, {"name": display, "present_in": 0}) + item["present_in"] += 1 + near = {} + for key, item in presence.items(): + squashed = _NOT_WORD.sub("", key) + if squashed: + near.setdefault(squashed, []).append(item["name"]) + similar = [{"names": names} for names in near.values() if len(names) > 1] + identical = len(groups) == 1 + if not inspected: + suggested = None + elif identical: + suggested = SCHEMA_POLICY_BY_NAME + else: + suggested = SCHEMA_POLICY_MAPPED if similar else SCHEMA_POLICY_UNION + return { + "tables": len(inspected), + "identical_columns": bool(inspected) and identical, + "same_order": bool(inspected) and identical and same_order, + "groups": list(groups.values()), + "columns": [{"name": item["name"], "present_in": item["present_in"]} for item in presence.values()], + "similar_columns": similar, + "suggested_policy": suggested, + } -def _cell_has_content(value): - return bool(value) and bool(str(value).strip()) +def _json_size(value): + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + + +def _fit_inspection_budget(report): + if _json_size(report) <= _INSPECT_MAX_BYTES: + return + for entry in report["sources"]: + for table in entry["tables"]: + if table.get("sample_rows"): + table["sample_rows"] = [] + report["samples_omitted"] = True + report["limitations"].append("Sample rows were left out to keep the inspection small.") + if _json_size(report) > _INSPECT_MAX_BYTES: + raise TabularMergeError( + "inspection_too_large", "The selected files have too many sheets or columns to inspect here.", + ) # --------------------------------------------------------------------------- -# Table readers +# Readers # --------------------------------------------------------------------------- @@ -611,9 +1753,8 @@ class _Table: """A header plus a row iterator over one sheet or CSV body.""" format = "" - - def describe(self): - return {} + sheet = None + header_row = None @property def header(self): @@ -626,43 +1767,103 @@ def close(self): return None -def _open_table(source, content, options, limits): +class _Reader: + """One opened source file; workbooks can hold several tables.""" + + format = "" + + def __init__(self, source): + self.source = source + + def describe(self): + return {} + + def sheet_states(self): + return [] + + def plan(self, options, limits): + """``(sheet, skip_reason)`` for each table to read, in order.""" + states = self.sheet_states() + if options.sheets == SHEETS_ALL: + if not states: + raise TabularMergeError("empty_source", f"{_safe_name(self.source)} has no worksheets.") + if len(states) > limits.max_sheets_per_source: + raise TabularMergeError( + "too_many_sheets", + f"{_safe_name(self.source)} has more than {limits.max_sheets_per_source} sheets.", + ) + return [(name, None if state == "visible" else "hidden_sheet") for name, state in states] + return [(_select_sheet(states, options.sheet, self.source), None)] + + def open_table(self, sheet, header_row): + raise NotImplementedError + + def close(self): + return None + + +def _open_reader(source, content, limits): table_format = TABULAR_MERGE_EXTENSION_FORMATS[source.resolved_extension()] if table_format == TABULAR_MERGE_FORMAT_CSV: - return _CsvTable(source, content) + return _CsvReader(source, content) if table_format == TABULAR_MERGE_FORMAT_XLSX: - return _XlsxTable(source, content, options.sheet, limits) - return _XlsTable(source, content, options.sheet) + return _XlsxReader(source, content, limits) + return _XlsReader(source, content) -class _CsvTable(_Table): +class _CsvReader(_Reader): format = TABULAR_MERGE_FORMAT_CSV def __init__(self, source, content): + super().__init__(source) + self._text, self._encoding = _decode_csv(content, source) + self._delimiter = _detect_delimiter(self._text) + + def describe(self): + return {"encoding": self._encoding, "delimiter": self._delimiter} + + def plan(self, options, limits): + return [(None, None)] + + def open_table(self, sheet, header_row): + return _CsvTable(self.source, self._text, self._delimiter, header_row) + + +class _CsvTable(_Table): + format = TABULAR_MERGE_FORMAT_CSV + + def __init__(self, source, text, delimiter, header_row): self._source = source - text, self._encoding = _decode_csv(content, source) - self._delimiter = _detect_delimiter(text) self._reader = csv.reader( - io.StringIO(text, newline=""), delimiter=self._delimiter, quotechar='"', - doublequote=True, strict=False, + io.StringIO(text, newline=""), delimiter=delimiter, quotechar='"', doublequote=True, strict=False, ) self._header = None - self._header_line = 0 + # Rows are counted as records, as a spreadsheet shows them, not as physical lines. + record = 0 try: for values in self._reader: + record += 1 + if header_row is not None and record < header_row: + continue if any(_cell_has_content(value) for value in values): self._header = values - self._header_line = self._reader.line_num + self.header_row = record + self._record = record break + if header_row is not None: + raise TabularMergeError( + "empty_source", f"Row {header_row} in {_safe_name(source)} is blank, so it has no column headers.", + ) except csv.Error as exc: raise TabularMergeError( "unreadable_csv", f"{_safe_name(source)} couldn't be read as CSV near line {self._reader.line_num}.", ) from exc if self._header is None: - raise TabularMergeError("empty_source", f"{_safe_name(source)} has no header row.") - - def describe(self): - return {"encoding": self._encoding, "delimiter": self._delimiter} + raise TabularMergeError( + "empty_source", + f"{_safe_name(source)} has no header row." if header_row is None + else f"{_safe_name(source)} has fewer than {header_row} rows.", + ) @property def header(self): @@ -672,7 +1873,8 @@ def rows(self): reader = self._reader try: for values in reader: - yield reader.line_num, values + self._record += 1 + yield self._record, values except csv.Error as exc: raise TabularMergeError( "unreadable_csv", f"{_safe_name(self._source)} couldn't be read as CSV near line {reader.line_num}.", @@ -726,11 +1928,11 @@ def _detect_delimiter(text): return best if counts[best] else "," -class _XlsxTable(_Table): +class _XlsxReader(_Reader): format = TABULAR_MERGE_FORMAT_XLSX - def __init__(self, source, content, sheet, limits): - self._source = source + def __init__(self, source, content, limits): + super().__init__(source) _guard_workbook_archive(content, source, limits) # openpyxl is imported only when a workbook is merged; it is a large binary dependency. from openpyxl import load_workbook @@ -743,39 +1945,67 @@ def __init__(self, source, content, sheet, limits): raise TabularMergeError( "unreadable_workbook", f"{_safe_name(source)} couldn't be opened as an Excel workbook.", ) from exc + + def sheet_states(self): try: - self._sheet = _select_sheet( - [(worksheet.title, getattr(worksheet, "sheet_state", "visible")) for worksheet in self._workbook.worksheets], - sheet, source, - ) - self._worksheet = self._workbook[self._sheet] - self._worksheet.reset_dimensions() - self._iterator = self._worksheet.iter_rows(values_only=True) - self._row_number = 0 - self._header = None - for values in self._iterator: - self._row_number += 1 - cells = [_excel_text(value) for value in values] - if any(_cell_has_content(value) for value in cells): - self._header = cells - break + return [ + (worksheet.title, getattr(worksheet, "sheet_state", "visible")) + for worksheet in self._workbook.worksheets + ] + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def open_table(self, sheet, header_row): + try: + return _XlsxTable(self.source, self._workbook[sheet], sheet, header_row) except TabularMergeError: - self.close() raise except Exception as exc: - self.close() raise TabularMergeError( - "unreadable_workbook", f"{_safe_name(source)} couldn't be read as an Excel workbook.", + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", ) from exc + + def close(self): + workbook = getattr(self, "_workbook", None) + if workbook is not None: + self._workbook = None + try: + workbook.close() + except Exception: + pass + + +class _XlsxTable(_Table): + format = TABULAR_MERGE_FORMAT_XLSX + + def __init__(self, source, worksheet, sheet, header_row): + self._source = source + self.sheet = sheet + worksheet.reset_dimensions() + self._iterator = worksheet.iter_rows(values_only=True) + self._row_number = 0 + self._header = None + for values in self._iterator: + self._row_number += 1 + if header_row is not None and self._row_number < header_row: + continue + cells = [_excel_text(value) for value in values] + if any(_cell_has_content(value) for value in cells): + self._header = cells + self.header_row = self._row_number + break + if header_row is not None: + raise TabularMergeError( + "empty_source", + f'Row {header_row} of sheet "{sheet}" in {_safe_name(source)} is blank, so it has no column headers.', + ) if self._header is None: - self.close() raise TabularMergeError( - "empty_source", f'Sheet "{self._sheet}" in {_safe_name(source)} has no header row.', + "empty_source", f'Sheet "{sheet}" in {_safe_name(source)} has no header row.', ) - def describe(self): - return {"sheet": self._sheet} - @property def header(self): return self._header @@ -792,15 +2022,6 @@ def rows(self): "unreadable_workbook", f"{_safe_name(self._source)} couldn't be read near row {self._row_number}.", ) from exc - def close(self): - workbook = getattr(self, "_workbook", None) - if workbook is not None: - self._workbook = None - try: - workbook.close() - except Exception: - pass - def _guard_workbook_archive(content, source, limits): if content[:8] == _OLE_MAGIC: @@ -823,11 +2044,11 @@ def _guard_workbook_archive(content, source, limits): raise TabularMergeError("workbook_too_large", f"{_safe_name(source)} is too large to merge once uncompressed.") -class _XlsTable(_Table): +class _XlsReader(_Reader): format = TABULAR_MERGE_FORMAT_XLS - def __init__(self, source, content, sheet): - self._source = source + def __init__(self, source, content): + super().__init__(source) if content[:8] != _OLE_MAGIC: raise TabularMergeError( "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel 97-2003 (.xls) workbook.", @@ -846,42 +2067,73 @@ def __init__(self, source, content, sheet): else f"{_safe_name(source)} couldn't be opened as an Excel workbook." ) raise TabularMergeError(code, message) from exc + + def sheet_states(self): try: names = self._book.sheet_names() - states = [] - for index, name in enumerate(names): - visibility = 0 - if sheet is None: - visibility = getattr(self._book.sheet_by_index(index), "visibility", 0) - states.append((name, "visible" if visibility == 0 else "hidden")) - self._sheet = _select_sheet(states, sheet, source) - self._worksheet = self._book.sheet_by_name(self._sheet) - self._datemode = self._book.datemode - self._next_row = 0 - self._header = None - while self._next_row < self._worksheet.nrows: - cells = self._row_values(self._next_row) - self._next_row += 1 - if any(_cell_has_content(value) for value in cells): - self._header = cells - break + visibility = getattr(self._book, "_sheet_visibility", None) + if not isinstance(visibility, list) or len(visibility) != len(names): + visibility = [ + getattr(self._book.sheet_by_index(index), "visibility", 0) for index in range(len(names)) + ] + return [(name, "visible" if state == 0 else "hidden") for name, state in zip(names, visibility)] + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def open_table(self, sheet, header_row): + try: + worksheet = self._book.sheet_by_name(sheet) + return _XlsTable(self.source, self._xlrd, worksheet, self._book.datemode, sheet, header_row) except TabularMergeError: - self.close() raise except Exception as exc: - self.close() raise TabularMergeError( - "unreadable_workbook", f"{_safe_name(source)} couldn't be read as an Excel workbook.", + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", ) from exc + + def close(self): + book = getattr(self, "_book", None) + if book is not None: + self._book = None + try: + book.release_resources() + except Exception: + pass + + +class _XlsTable(_Table): + format = TABULAR_MERGE_FORMAT_XLS + + def __init__(self, source, xlrd, worksheet, datemode, sheet, header_row): + self._source = source + self._xlrd = xlrd + self._worksheet = worksheet + self._datemode = datemode + self.sheet = sheet + self._next_row = 0 + self._header = None + while self._next_row < worksheet.nrows: + row_number = self._next_row + 1 + cells = self._row_values(self._next_row) + self._next_row += 1 + if header_row is not None and row_number < header_row: + continue + if any(_cell_has_content(value) for value in cells): + self._header = cells + self.header_row = row_number + break + if header_row is not None: + raise TabularMergeError( + "empty_source", + f'Row {header_row} of sheet "{sheet}" in {_safe_name(source)} is blank, so it has no column headers.', + ) if self._header is None: - self.close() raise TabularMergeError( - "empty_source", f'Sheet "{self._sheet}" in {_safe_name(source)} has no header row.', + "empty_source", f'Sheet "{sheet}" in {_safe_name(source)} has no header row.', ) - def describe(self): - return {"sheet": self._sheet} - @property def header(self): return self._header @@ -912,15 +2164,6 @@ def _row_values(self, row_index): values.append(_excel_text(cell.value)) return values - def close(self): - book = getattr(self, "_book", None) - if book is not None: - self._book = None - try: - book.release_resources() - except Exception: - pass - def _select_sheet(sheets: List[Tuple[str, str]], requested, source): names = [name for name, _ in sheets] @@ -975,20 +2218,52 @@ def _excel_text(value): __all__ = [ + "DEDUPE_EXACT_ROWS", + "DEDUPE_KEEP_FIRST", + "DEDUPE_KEEP_LAST", + "DEDUPE_KEY_COLUMNS", + "DEDUPE_NONE", + "DEFAULT_INSPECT_SAMPLE_ROWS", + "DEFAULT_SHEET_COLUMN_NAME", "DEFAULT_SOURCE_COLUMN_NAME", + "MAX_INSPECT_SAMPLE_ROWS", + "ON_INCOMPATIBLE_EXCLUDE", + "ON_INCOMPATIBLE_FAIL", "SCHEMA_POLICY_BY_NAME", "SCHEMA_POLICY_EXACT_ORDER", + "SCHEMA_POLICY_MAPPED", + "SCHEMA_POLICY_UNION", + "SHEETS_ALL", + "SHEETS_FIRST", + "SORT_TYPE_DATE", + "SORT_TYPE_NUMBER", + "SORT_TYPE_TEXT", + "TABULAR_COLUMN_MAPPING_PROFILE", + "TABULAR_INSPECTION_VERSION", + "TABULAR_MERGE_DEDUPE_KEEP", + "TABULAR_MERGE_DEDUPE_MODES", + "TABULAR_MERGE_INCOMPATIBLE_MODES", "TABULAR_MERGE_REPORT_VERSION", "TABULAR_MERGE_SCHEMA_POLICIES", + "TABULAR_MERGE_SHEET_MODES", + "TABULAR_MERGE_SORT_TYPES", "TABULAR_MERGE_SUPPORTED_EXTENSIONS", + "TabularInspectOptions", "TabularMergeCancelled", "TabularMergeError", "TabularMergeLimits", "TabularMergeOptions", "TabularMergeResult", + "TabularMergeSort", "TabularMergeSource", + "inspect_tabular_sources", "iter_csv_rows", "merge_tabular_sources", "normalize_header_name", + "squash_header_name", + "tabular_column_mapping_schema", + "tabular_inspect_options_from_arguments", + "tabular_mapping_from_profile", "tabular_merge_format_for_file_name", + "tabular_merge_options_from_arguments", ] diff --git a/application/single_app/templates/admin/_panes/actions.html b/application/single_app/templates/admin/_panes/actions.html index bd747b705..e447fa0cf 100644 --- a/application/single_app/templates/admin/_panes/actions.html +++ b/application/single_app/templates/admin/_panes/actions.html @@ -124,7 +124,7 @@
Document Action Capabilities - +
diff --git a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx index 7c8e88f0f..fb9c75669 100644 --- a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx @@ -64,6 +64,7 @@ import type { import type { ImageReferenceRequest } from '../../lib/imageReferences'; import { useDocumentTitles } from '../../lib/documentTitles'; import { plannedFileSpecification, type OrchestrationExportFormat } from '../../lib/orchestrationExports'; +import { tabularArgumentEntries } from '../../lib/orchestrationMerge'; import { WORKFLOW_RESULTS_ARGUMENT_KEYS, WORKFLOW_RESULTS_CAPABILITY, @@ -104,6 +105,11 @@ const STATUS_LABELS: Record = { /** A step argument key/value the run will use, minus the document fields shown as chips. */ function readableArguments(step: OrchestrationStep): Array<[string, string]> { + // Spreadsheet merge and inspection settings are put in words, without restating defaults. + const tabular = tabularArgumentEntries(step.capability_id, step.arguments); + if (tabular) { + return tabular; + } const hidden = new Set([ 'document_ids', 'right_document_ids', @@ -429,7 +435,7 @@ export function OrchestrationRunView({ ) : null} {args.length > 0 ? ( -
+
{args.map(([key, value]) => (
diff --git a/application/v2_ui/src/lib/orchestrationMerge.ts b/application/v2_ui/src/lib/orchestrationMerge.ts new file mode 100644 index 000000000..1a76b2960 --- /dev/null +++ b/application/v2_ui/src/lib/orchestrationMerge.ts @@ -0,0 +1,148 @@ +// orchestrationMerge.ts +// How the plan card states the settings of a spreadsheet merge or inspection step. +// +// tabular_merge and tabular_inspect arguments are enums, column lists and small objects. The +// card shows each one in words, leaves out values that only restate a default, and treats +// column and sheet names as data. A value the server would refuse is not put in words. + +/** The orchestration capability that merges CSV and Excel files. */ +export const TABULAR_MERGE_CAPABILITY = 'tabular_merge'; + +/** The orchestration capability that inspects CSV and Excel files before a merge. */ +export const TABULAR_INSPECT_CAPABILITY = 'tabular_inspect'; + +const SCHEMA_POLICY_WORDS: Record = { + by_name: 'Same columns, in any order', + exact_order: 'Same columns, in the same order', + union: 'Keep every column from every file', + mapped: 'Only the listed columns', +}; + +const SORT_TYPE_WORDS: Record = { + text: 'text', + number: 'number', + date: 'date', +}; + +const DOC_SCOPE_WORDS: Record = { + personal: 'Personal workspace', + group: 'Group workspaces', + public: 'Public workspaces', +}; + +const DEFAULT_SOURCE_COLUMN = 'Source File'; + +function has(record: Record, key: unknown): key is string { + return typeof key === 'string' && Object.prototype.hasOwnProperty.call(record, key); +} + +function names(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === 'string' && item.trim() !== ''); +} + +function aliasWords(value: unknown): string { + if (!value || typeof value !== 'object' || Array.isArray(value)) return ''; + return Object.entries(value as Record) + .map(([target, aliases]) => { + const others = names(aliases); + return target.trim() && others.length ? `${target} \u2190 ${others.join(', ')}` : ''; + }) + .filter(Boolean) + .join('; '); +} + +function sortWords(value: unknown): string { + if (!Array.isArray(value)) return ''; + const keys: string[] = []; + for (const item of value) { + if (!item || typeof item !== 'object' || Array.isArray(item)) return ''; + const key = item as Record; + if (typeof key.column !== 'string' || !key.column.trim()) return ''; + if (key.value_type !== undefined && !has(SORT_TYPE_WORDS, key.value_type)) return ''; + const details = [ + has(SORT_TYPE_WORDS, key.value_type) && key.value_type !== 'text' ? SORT_TYPE_WORDS[key.value_type] : '', + key.descending === true ? 'descending' : '', + ].filter(Boolean); + keys.push(details.length ? `${key.column} (${details.join(', ')})` : key.column); + } + return keys.join(', '); +} + +function tableEntries(args: Record): Array<[string, string]> { + const entries: Array<[string, string]> = []; + if (has(DOC_SCOPE_WORDS, args.doc_scope)) { + entries.push(['workspace', DOC_SCOPE_WORDS[args.doc_scope]]); + } + if (typeof args.sheet === 'string' && args.sheet.trim()) { + entries.push(['sheet', args.sheet]); + } else if (args.sheets === 'all') { + entries.push(['sheets', 'Every visible sheet']); + } + if (typeof args.header_row === 'number' && Number.isInteger(args.header_row) && args.header_row >= 1) { + entries.push(['header row', `Row ${args.header_row}`]); + } + return entries; +} + +function mergeEntries(args: Record): Array<[string, string]> { + const entries: Array<[string, string]> = []; + if (has(SCHEMA_POLICY_WORDS, args.schema_policy)) { + entries.push(['columns', SCHEMA_POLICY_WORDS[args.schema_policy]]); + } + const kept = names(args.columns); + if (kept.length) { + entries.push(['keep', kept.join(', ')]); + } + const aliases = aliasWords(args.column_aliases); + if (aliases) { + entries.push(['same column as', aliases]); + } + entries.push(...tableEntries(args)); + if (args.include_source_column === false) { + entries.push(['file name column', 'Off']); + } else if (typeof args.source_column_name === 'string' && args.source_column_name.trim() + && args.source_column_name.trim() !== DEFAULT_SOURCE_COLUMN) { + entries.push(['file name column', args.source_column_name]); + } + if (args.on_incompatible === 'exclude') { + entries.push(['files that don\u2019t fit', 'Left out and reported']); + } + const keep = args.dedupe_keep === 'last' ? 'keeping the last' : 'keeping the first'; + if (args.dedupe === 'exact_rows') { + entries.push(['duplicates', `Remove identical rows, ${keep}`]); + } else if (args.dedupe === 'key_columns') { + const keys = names(args.dedupe_columns); + if (keys.length) { + entries.push(['duplicates', `Remove rows with the same ${keys.join(', ')}, ${keep}`]); + } + } + const order = sortWords(args.sort_by); + if (order) { + entries.push(['sort by', order]); + } + return entries; +} + +function inspectEntries(args: Record): Array<[string, string]> { + const entries = tableEntries(args); + if (typeof args.sample_rows === 'number' && Number.isInteger(args.sample_rows) + && args.sample_rows >= 0 && args.sample_rows <= 10) { + entries.push(['sample rows', String(args.sample_rows)]); + } + return entries; +} + +/** + * The arguments of a merge or inspection step in words, or null for any other capability. + * Document selections are shown as chips by the card, so they are never listed here. + */ +export function tabularArgumentEntries(capabilityId: string, args: unknown): Array<[string, string]> | null { + if (capabilityId !== TABULAR_MERGE_CAPABILITY && capabilityId !== TABULAR_INSPECT_CAPABILITY) { + return null; + } + const record = args && typeof args === 'object' && !Array.isArray(args) + ? args as Record + : {}; + return capabilityId === TABULAR_MERGE_CAPABILITY ? mergeEntries(record) : inspectEntries(record); +} diff --git a/docs/admin/agents-actions.md b/docs/admin/agents-actions.md index 8b9678edc..5e811bcb7 100644 --- a/docs/admin/agents-actions.md +++ b/docs/admin/agents-actions.md @@ -201,10 +201,12 @@ full, so an answer covers all of them rather than only the passages a search returned; Comparison reads one baseline document against the others, which is what answers questions about what changed between versions. -**Merge** combines the rows of several CSV or Excel files that share the same -columns into one table. Since **0.261.218**, V2 chat orchestration uses it when -someone asks to merge, combine or append spreadsheets, and the plan delivers the -result as a CSV or Excel file. Code appends the rows; no model reads them, so a +**Merge** combines the rows of several CSV or Excel files into one table. Since +**0.261.218**, V2 chat orchestration uses it when someone asks to merge, combine +or append spreadsheets, and the plan delivers the result as a CSV or Excel file. +Since **0.261.219** it also reconciles files whose columns differ, and the same +switch and file limit govern **Inspect spreadsheets**, which reads the files' +headers and samples first. Code appends the rows; no model reads them, so a merge costs far less than analyzing the same files. That is why its limits are higher, and why it is limited by merged rows as well as by files: a few very large files can cost more than many small ones. @@ -227,8 +229,8 @@ than as separate settings. | Enable Document Comparison | Offers Document Comparison in the Action menu. | On | — | `document_action_capabilities.comparison.enabled` | | Comparison: Chat Document Limit | Most documents one chat message may compare, including the baseline. | 3 | 2–300 | `comparison.chat_max_documents` | | Comparison: Workflow Document Limit | The same limit for a workflow run. | 10 | 2–1000 | `comparison.workflow_max_documents` | -| Enable Merge | Lets chat orchestration combine same-structure CSV and Excel files into one file. | On | — | `document_action_capabilities.merge.enabled`; also governed by the **Merge spreadsheets** orchestration capability. | -| Merge: Chat Document Limit | Most files one chat request may merge. | 10 | 2–300 | `merge.chat_max_documents` | +| Enable Merge | Lets chat orchestration combine CSV and Excel files into one file, and inspect their columns first. | On | — | `document_action_capabilities.merge.enabled`; also governed by the **Merge spreadsheets** and **Inspect spreadsheets** orchestration capabilities. | +| Merge: Chat Document Limit | Most files one chat request may merge or inspect. | 10 | 2–300 | `merge.chat_max_documents` | | Merge: Workflow Document Limit | The same limit for a workflow run. | 100 | 2–1000 | `merge.workflow_max_documents` | | Merge: Chat Row Limit | Most rows one chat merge may produce. A merge over the limit stops without creating a file. | 250,000 | 1,000–1,000,000 | `merge.chat_max_rows` | | Merge: Workflow Row Limit | The same row limit for a workflow run. | 1,000,000 | 1,000–1,000,000 | `merge.workflow_max_rows` | diff --git a/docs/admin/orchestration.md b/docs/admin/orchestration.md index eafc6b4d1..63c650fad 100644 --- a/docs/admin/orchestration.md +++ b/docs/admin/orchestration.md @@ -229,17 +229,30 @@ changes nothing, so it never makes a plan wait for approval. Clearing it keeps saved results out of plans; **Ask in chat** on a run is not affected. Since **0.261.218**, **Merge spreadsheets** (`tabular_merge`) combines the rows -of two or more CSV or Excel files that share the same columns into one table, -which the plan then delivers as a CSV or Excel file through **Create a file**. -Code appends every row exactly, in the order the files are listed, and adds a -**Source File** column by default; no model reads or rewrites the rows, so -codes such as `007` keep their leading zeros. Columns may appear in any order. -Files whose columns differ are not merged: the run says so and nothing is -created. Merge never matches rows on a key column the way a lookup or join -would. It's offered while **Enable Merge** is on under +of two or more CSV or Excel files into one table, which the plan then delivers as +a CSV or Excel file through **Create a file**. Code appends every row exactly, in +the order the files are listed, and adds a **Source File** column by default; no +model reads or rewrites the rows, so codes such as `007` keep their leading +zeros. Columns may appear in any order. Since **0.261.219**, a merge can also +reconcile files whose columns differ: keep every column, treat differently named +headers as one column, keep only listed columns, leave out files or sheets that +don't fit, read headers below a title row, merge every sheet of a workbook, +remove duplicate rows and sort. Without one of those choices, files whose +columns differ are not merged: the run says so and nothing is created. Merge +never matches rows on a key column the way a lookup or join would. + +**Inspect spreadsheets** (`tabular_inspect`), since **0.261.219**, reads the +sheet names, headers, row counts and a few sample rows of CSV or Excel files and +reports how their columns line up, without changing anything. Plans use it to +answer questions about files' structure, and before a merge whose columns may +differ, so **Prepare content** can line up the columns as a column mapping that +the merge checks and applies with code. + +Both are offered while **Enable Merge** is on under [Document Action Capabilities]({{ '/admin/agents-actions/' | relative_url }}#document-action-capabilities-card), -which also sets how many files and rows one chat request may merge. Clearing -it here keeps merges out of plans. See +which also sets how many files and rows one chat request may merge or inspect. +Clearing either here keeps it out of plans; a narrowed list needs **Inspect +spreadsheets** and **Prepare content** for merges that line up columns. See [Merge files in chat]({{ '/guides/merge-files/' | relative_url }}). Selecting a capability does not grant source/model access or enable another diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index f088e2ef1..d18ce42b5 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -130,7 +130,8 @@ permissions cannot be supplied as bindings. | `document_analyze` | Narrative `findings: records-v1` and `coverage: structured-v1`; public `records` and a Markdown `report` are optional outputs only when genuinely produced. | | `document_compare` | Narrative `comparison: comparison-v1` and `coverage: structured-v1`; a Markdown `report` is optional when available. | | `tabular_analyze` | Server-bound native query/transform returns `records` and `coverage`; analysis-only returns `analysis` and `coverage`; combined transformation/analysis returns all three. | -| `tabular_merge` | Since **0.261.218**: `records: records-v1` holding every row of two or more same-structure CSV or Excel sources, appended by code in source order, and `report: structured-v1` (`tabular-merge-report-v1`) with the policy, columns, per-source rows and limitations. Coverage counts sources. See [V2 file merge](V2_FILE_MERGE.md). | +| `tabular_merge` | Since **0.261.218**: `records: records-v1` holding every row of two or more CSV or Excel sources, appended by code in source order, and `report: structured-v1` (`tabular-merge-report-v1`) with the policy, columns, per-source rows and limitations. Coverage counts sources. Since **0.261.219** columns may be reconciled (union, mapped, aliases, or a bound `mapping` from a compose step using profile `tabular_column_mapping_v1`), and columns some sources lack are nullable. See [V2 file merge](V2_FILE_MERGE.md). | +| `tabular_inspect` | Since **0.261.219** (Gather): `inspection: structured-v1` (`tabular-inspection-v1`) with each source's sheets, headers, row counts, samples and problems, and how the sources' columns line up. Coverage counts sources. See [Phase 2](V2_FILE_MERGE_PHASE_2_RECONCILIATION.md). | | `compose` | Explicitly named text, Markdown, records or structured values. There is no default output declaration. | | Server-bound external Gather | `prepared: structured-v1` with exact returned content and separately authorized external-source lineage. | diff --git a/docs/explanation/features/V2_FILE_MERGE.md b/docs/explanation/features/V2_FILE_MERGE.md index 16afc2265..60687ffb3 100644 --- a/docs/explanation/features/V2_FILE_MERGE.md +++ b/docs/explanation/features/V2_FILE_MERGE.md @@ -1,8 +1,9 @@ # V2 File Merge -Version: **0.261.218** +Version: **0.261.219** -Implemented in version: **0.261.218** (Phase 1, same-structure spreadsheets), recorded in +Implemented in version: **0.261.218** (Phase 1, same-structure spreadsheets) and +**0.261.219** (Phase 2, reconciling different structures), recorded in `application/single_app/config.py`. GitHub issue: [#1619](https://github.com/microsoft/simplechat/issues/1619) @@ -24,7 +25,7 @@ request and the easiest to verify exactly; documents and decks follow. | Phase | Scope | Status | | --- | --- | --- | | 1 | Same-structure CSV and Excel merge in V2 chat (`tabular_merge`), Merge limits, exact-schema CSV/XLSX exports | Implemented in **0.261.218** — see [Phase 1](V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md) | -| 2 | Reconciling different structures: column inspection, union and mapped policies, sheet modes, duplicate removal, AI-suggested column mapping | Planned | +| 2 | Reconciling different structures: column inspection (`tabular_inspect`), union and mapped policies, aliases, header rows, every-sheet mode, exclusion, duplicate removal, sorting, AI-prepared column mapping | Implemented in **0.261.219** — see [Phase 2](V2_FILE_MERGE_PHASE_2_RECONCILIATION.md) | | 3 | Large and recurring merges (100+ files) as V2 workflow tasks | Planned | | 4 | Document assembly foundation, PDF merge, and workbooks with one tab per file | Planned | | 5 | Word merge, keeping each document's styles or the first document's | Planned | @@ -39,8 +40,8 @@ plans the work. | Purpose | Role in a merge | | --- | --- | -| Gather (optional) | `document_search` finds the files when the user describes them instead of selecting them. Its `sources` output binds into the merge. | -| Reason | `tabular_merge` reads the authorized files and appends their rows with code into one retained table. No model reads or rewrites rows. | +| Gather (optional) | `document_search` finds the files when the user describes them instead of selecting them. Its `sources` output binds into the merge. `tabular_inspect` reads the files' sheets, headers, row counts and samples and reports how their columns line up. | +| Reason | `compose` can prepare a `tabular_column_mapping_v1` column mapping from an inspection. `tabular_merge` reads the authorized files and appends their rows with code into one retained table. No model reads or rewrites rows. | | Render | `render_file` turns the retained table into a CSV or Excel file. | Merging is Reason, not Render, because it computes a new dataset from sources; Render only @@ -56,27 +57,34 @@ saved instruction such as "always give me Excel" shapes merge requests without e | File | Responsibility | | --- | --- | -| `functions_tabular_merge.py` | Pure merge engine: CSV, XLSX/XLSM and XLS readers that keep values as text, header normalization, schema policies, provenance column, limits, cancellation, archive guards, and the merge report. No Flask, settings, storage or model imports. | -| `functions_orchestration_merge.py` | Orchestration glue: Merge limits from settings, the authorized-source manifest check, screening-aware byte loaders, persistence of the retained `records` and `report`, and the engine-to-failure-code map. | -| `functions_orchestration_adapters.py` | `run_tabular_merge`, registered in `ADAPTER_REGISTRY`. | -| `functions_orchestration_registry.py` | The `tabular_merge` capability descriptor and the `merge` document-action gate. | -| `functions_orchestration_schema.py` | Plan rules: explicit document IDs or a bound source set, tabular-only sources, the Merge chat limit, and the merge failure messages. | +| `functions_tabular_merge.py` | Pure merge and inspection engine: CSV, XLSX/XLSM and XLS readers that keep values as text, header normalization, schema policies and aliases, header rows, sheet modes, provenance columns, exclusion, duplicate removal, bounded sort, limits, cancellation, archive guards, the merge report, the inspection, and the column mapping profile. No Flask, settings, storage or model imports. | +| `functions_orchestration_merge.py` | Orchestration glue: Merge limits from settings, the authorized-source manifest check, screening-aware byte loaders, persistence of the retained `records`, `report` and `inspection`, and the engine-to-failure-code maps. | +| `functions_orchestration_adapters.py` | `run_tabular_inspect` and `run_tabular_merge`, registered in `ADAPTER_REGISTRY`. | +| `functions_orchestration_registry.py` | The `tabular_inspect` and `tabular_merge` capability descriptors and the `merge` document-action gate. | +| `functions_orchestration_schema.py` | Plan rules: explicit document IDs or a bound source set, tabular-only sources, the Merge chat limit, valid option combinations, the mapping producer, and the merge and inspection failure messages. | +| `functions_orchestration_services.py` | The `tabular_column_mapping_v1` prepared-content profile and its validator. | | `functions_document_actions.py` | The `merge` document action: enablement, file limits and row limits for chat and workflows. | | `functions_generated_export_registry.py`, `functions_structured_file_renderers.py`, `functions_generated_office_adapters.py` | The `exact_tabular_records_v1` (CSV) and `exact_tabular_workbook_v1` (XLSX) profiles. | +| `application/v2_ui/src/lib/orchestrationMerge.ts` | The plan review's wording for merge and inspection settings. | ### Result contract `tabular_merge` retains two outputs under contract `tabular-merge-v1`: - `records` (`records-v1`): one string column per merged column, in output order, with - every row. At most 256 columns, because retained record schemas allow 256. + every row. At most 256 columns, because retained record schemas allow 256. A column is + nullable when some merged file or sheet doesn't have it. - `report` (`structured-v1`): `tabular-merge-report-v1`, holding the policy, the output - columns, the provenance column name, totals, one entry per source (file name, format, - sheet, encoding, delimiter, rows, skipped blank rows, padded short rows, column - differences, warnings) and limitations. + columns, nullable columns, the file and sheet name columns, totals, one entry per file + or sheet (file name, format, sheet, header row, encoding, delimiter, status and reason, + rows, skipped blank rows, padded short rows, removed duplicates, column differences, + renamed and left-out columns, warnings), any prepared column mapping, and limitations. -Coverage counts sources. The result is complete only when every source merged; a merge -either finishes or produces nothing. +Coverage counts sources. The result is complete only when every source merged or was +left out by request; a merge either finishes or produces nothing. + +`tabular_inspect` retains `inspection` (`structured-v1`, `tabular-inspection-v1`) under +contract `tabular-inspect-v1`. ### Limits @@ -91,6 +99,9 @@ either finishes or produces nothing. | Header name | 256 UTF-8 bytes | Retained column name limit | | Cell | 32,767 characters | Excel's cell limit | | Workbook expansion | 512 MB uncompressed, 10,000 parts | Engine guard | +| Sheets per workbook (every-sheet mode) | 100 | Engine constant | +| Rows that can be sorted | 250,000 | Engine constant, capped by the row limit | +| Inspection size | 96 KB, samples dropped first | Engine constant | Rendering keeps its own limits: CSV exports hold at most 1,000,000 records, and XLSX at most 1,048,575 rows, 5,000,000 cells and 32 MiB. CSV suits the largest merges. @@ -120,13 +131,17 @@ and the orchestration Capabilities list | Test | Covers | | --- | --- | | `functional_tests/test_tabular_merge_engine.py` | Readers, encodings, delimiters, Excel values, sheet selection, legacy XLS conversion, policies, mismatch reporting, provenance, limits, guards, cancellation, spill to disk at 120,000 rows. | +| `functional_tests/test_tabular_merge_reconciliation.py` | Union, mapped and alias policies, header rows, every-sheet mode, exclusion, duplicates, sorting and inspection. | | `functional_tests/test_orchestration_tabular_merge_capability.py` | Registry and gating, plan validation, real executor runs without model calls, search-bound sources, failure messages, revoked access, and CSV/XLSX rendering of the retained result. | +| `functional_tests/test_orchestration_tabular_reconciliation.py` | Inspection, option combinations, the mapping producer, inspect → compose mapping → merge through the real executor, nullable union columns, and the planner recipe. | +| `functional_tests/test_v2_orchestration_merge_arguments.mjs`, `ui_tests/test_v2_orchestration_merge_arguments.py` | The plan review's wording for merge and inspection settings. | | `functional_tests/test_v2_admin_actions_parity.py`, `functional_tests/test_admin_settings_pane_variable_scope.py` | Admin field paths, bounds and defaults, and the admin card rendering with and without stored Merge settings. | ## Known limitations - Merge appends rows. It does not match rows across files on a key column. -- Phase 1 requires the same columns in every file; Phase 2 reconciles different ones. +- Column aliases apply to every file; a header that means different things in different + files can't be mapped per file. - Excel formulas contribute their last calculated values; a workbook saved without them merges those cells as blanks. - Workflows cannot run merges until Phase 3. diff --git a/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md b/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md index 80f573c96..aabe7546d 100644 --- a/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md +++ b/docs/explanation/features/V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md @@ -5,7 +5,8 @@ Version: **0.261.218** Implemented in version: **0.261.218**, recorded in `application/single_app/config.py`. GitHub issue: [#1619](https://github.com/microsoft/simplechat/issues/1619). Part of -[V2 File Merge](V2_FILE_MERGE.md). +[V2 File Merge](V2_FILE_MERGE.md). [Phase 2](V2_FILE_MERGE_PHASE_2_RECONCILIATION.md) +(0.261.219) builds on this phase to reconcile files whose columns differ. Dependencies: Chat Orchestration (V2), at least one workspace type, `openpyxl` and `xlrd` (already in `requirements.txt`). diff --git a/docs/explanation/features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md b/docs/explanation/features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md new file mode 100644 index 000000000..1ac49dc27 --- /dev/null +++ b/docs/explanation/features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md @@ -0,0 +1,230 @@ +# V2 File Merge — Phase 2: Reconciling Different Structures + +Version: **0.261.219** + +Implemented in version: **0.261.219**, recorded in `application/single_app/config.py`. + +GitHub issue: [#1619](https://github.com/microsoft/simplechat/issues/1619). Umbrella document: +[V2 File Merge](V2_FILE_MERGE.md). Builds on +[Phase 1](V2_FILE_MERGE_PHASE_1_SPREADSHEETS.md). + +Dependencies: no new packages. The engine uses the standard library plus the existing +`openpyxl` and `xlrd` readers; the column mapping uses the existing compose step and its +prepared-content profile validator. + +## Overview + +Phase 1 merged only files with the same columns. Real exports drift: a column is renamed +(`cust_id` becomes `Customer ID`), a file gains a column, a workbook has a title above its +headers, or one sheet in a workbook holds notes instead of data. Phase 2 lets a merge +reconcile those differences while staying deterministic: + +- **Inspect first.** A new Gather capability, `tabular_inspect`, reads each file's sheets, + headers, row counts and sample rows and reports how the files line up, including near + matches and a suggested policy. +- **Choose a policy.** `union` keeps every column; `mapped` keeps exactly a requested + column list; `column_aliases` renames other header names to one column under any policy. +- **Let a model suggest the mapping, but not perform it.** A compose step prepares a + `tabular_column_mapping_v1` value from the inspection. The merge validates it again and + code applies it, so a model never copies or rewrites rows. +- **Leave out what doesn't fit.** `on_incompatible: exclude` reports and skips files or + sheets whose columns don't fit instead of failing, and the result says so. +- **Shape the table.** `header_row`, `sheets: all` with a **Source Sheet** column, duplicate + removal by exact row or key columns, and a bounded multi-column sort. + +## Technical specifications + +### Architecture + +```mermaid +flowchart LR + I[tabular_inspect
Gather] -->|inspection| C[compose
profile tabular_column_mapping_v1] + C -->|mapping| M[tabular_merge
Reason, mapped policy] + M -->|records| R[render_file
csv / xlsx exact profiles] + I -. review-first: compose an answer, merge next turn .-> A[answer] +``` + +The one-shot recipe is inspect → compose mapping → merge → render. The review-first recipe +composes an answer from the inspection and merges in a later turn, with explicit +`columns` and `column_aliases`. Both recipes are offered to the planner only when +`tabular_inspect`, `tabular_merge`, `compose` and `render_file` are available. + +### Components + +| File | Change | +| --- | --- | +| `functions_tabular_merge.py` | Union and mapped policies, aliases, header row, all-sheets mode, exclusion, duplicate removal, bounded sort, inspection, the mapping profile schema and validator, and the shared argument-to-options conversion used by plan validation and the adapter. | +| `functions_orchestration_merge.py` | Inspection persistence (Gather role), nullable record columns, the mapping recorded in the merge report, step summaries, and the inspection failure-code map. | +| `functions_orchestration_adapters.py` | `run_tabular_inspect`; `run_tabular_merge` reads and re-validates a bound `mapping` input. Both share `_tabular_step_failure`. | +| `functions_orchestration_registry.py` | The `tabular_inspect` descriptor and the extended `tabular_merge` inputs and `mapping` input kind. | +| `functions_orchestration_schema.py` | Plan rules for inspect sources, merge option combinations, the mapping producer, tabular-only sources for inspect, and new failure messages. | +| `functions_orchestration_services.py` | Registers `tabular_column_mapping_v1` in `composition_profiles()` and validates it in `validate_composition_profile`. | +| `functions_orchestration_executor.py` | Binds a search's source set into inspect steps, as for merge and analyze. | +| `functions_orchestration_deliverables.py`, `functions_orchestration_planner.py` | The reconciliation recipe and planner guidance. | +| `application/v2_ui/src/lib/orchestrationMerge.ts`, `OrchestrationRunView.tsx` | The plan review states merge and inspection settings in words. | +| `admin_settings_fields.py` | **Inspect spreadsheets** in the orchestration Capabilities list. | + +### `tabular_inspect` (Gather) + +| Input | Meaning | +| --- | --- | +| `document_ids` (1 or more) or bound `inputs.sources` | The files, in order. | +| `sheet` / `sheets` (`first` or `all`) | Which sheets to read. | +| `header_row` (1–1000) | The row holding the headers; omitted means the first nonblank row. | +| `sample_rows` (0–10, default 3) | Sample rows kept per sheet. | + +It retains `inspection` (`structured-v1`, `tabular-inspection-v1`, contract +`tabular-inspect-v1`): per file its format, encoding and delimiter, every sheet with its +visibility, and per inspected sheet the header row used, columns, non-blank row count +(capped at the chat row limit), samples clipped to 80 characters, and warnings such as a +likely title row with the suggested `header_row`. A file that can't be read is reported +with its problem instead of failing the step. `compatibility` holds identical-column +groups, every column with how many sheets have it, near matches (names equal after +ignoring case, spaces, punctuation and underscores) and a suggested policy: `by_name` +when every sheet has the same columns, `mapped` when near matches exist, otherwise +`union`. The value is kept under 96 KB by dropping samples first. + +Inspection is gated by the Merge document action and uses its chat file limit. + +### `tabular_merge` inputs added in Phase 2 + +| Input | Meaning | +| --- | --- | +| `schema_policy` | `by_name` (default), `exact_order`, `union` or `mapped`. | +| `columns` | `mapped` only: the output columns in order (up to 255). | +| `column_aliases` | `{column: [other header names]}`, up to 256 columns and 32 names each. An alias may not be another column's name or belong to two columns. | +| `mapping` input | A compose step's `tabular_column_mapping_v1` output. It makes the policy `mapped` and can't be combined with `columns` or `column_aliases`. | +| `header_row`, `sheet`, `sheets` | As for inspection. `sheets: all` adds a **Source Sheet** column when the file name column is on. | +| `on_incompatible` | `fail` (default) or `exclude`. | +| `dedupe`, `dedupe_columns`, `dedupe_keep` | `none` (default), `exact_rows` or `key_columns` (1–16 columns); keep `first` (default) or `last`. | +| `sort_by` | Up to three `{column, descending, value_type}` keys; `value_type` is `text`, `number` or `date`. | + +The new inputs have no schema defaults, so a plan only lists what the planner chose and +the plan review stays short. Plan validation builds the same engine options the adapter +builds, so a combination the engine would refuse is refused at planning with +`merge_options_invalid`. A bound `mapping` must come from a compose step whose output is +`structured-v1` with profile `tabular_column_mapping_v1` (`mapping_profile_required`); a +retained-result alias is checked when the merge runs. + +### Policy semantics + +- **union**: the first file's columns, then each new column in the order a later file + introduces it. Rows from files without a column hold `null`. A later column whose name + collides with the file or sheet name column is renamed with a numeric suffix and a + warning. +- **mapped**: exactly `columns`. Headers that match a column or one of its aliases fill + it; other headers are left out and listed per source as `ignored_columns`. A file that + fills none of the columns is incompatible. +- **by_name / exact_order** keep Phase 1 behavior, with aliases applied first. +- **Exclusion** applies to files or sheets whose columns differ (`columns_differ`), that + fill none of the mapped columns (`no_matching_columns`), that have no header row, or + that lack a named sheet. Unreadable or encrypted files still fail the merge. +- **all sheets** skips hidden sheets and sheets without a header row; each is listed in + the report with its reason. + +### Duplicates and sort + +Duplicate detection compares the data columns, never the file or sheet name columns, using +a 128-bit BLAKE2b digest of the values. Missing and empty values compare equal and +trailing blanks are ignored, so a row read before a later file added a column matches the +same row read after it. Rows whose key values are all blank are never duplicates. +Keeping the first copy drops later copies as they are read; keeping the last copy records +the last position of each key and filters on read-back, adjusting each source's row +count. + +Sorting is stable and multi-key. Numbers accept thousands separators and a leading `$`, +`€`, `£` or `¥`; dates accept ISO 8601 (Excel dates are already ISO). Values that aren't +the requested type sort after typed values as case-insensitive text, and blanks sort +last in either direction. Only the sort keys and byte offsets stay in memory, and rows are +read back from the spool in order; sorting is limited to 250,000 rows +(`sort_limit_exceeded`). + +### Column mapping profile + +`tabular_column_mapping_v1` is offered through `composition_profiles()` like the slide +deck profile: + +```json +{ + "columns": ["Customer ID", "Name", "Amount"], + "mappings": [ + {"source_column": "cust_id", "target_column": "Customer ID", "confidence": "high"}, + {"source_column": "Total", "target_column": "Amount", "confidence": "low"}, + {"source_column": "Notes", "target_column": null, "confidence": "high"} + ], + "notes": "Lined up the customer columns." +} +``` + +The validator refuses unknown fields, repeated columns or headers, targets that aren't +listed columns, and a header that is itself a different listed column. Compose asks the +model once more when a reply breaks the profile (`result_schema_invalid`), then fails the +step. The merge validates the retained value again, converts it to `columns` and +`column_aliases`, and records in the report which headers were explicitly left out, +which mappings were low confidence, and the notes. + +### Result changes + +- `records` columns are `nullable` when some merged file or sheet doesn't have them. +- The report adds `nullable_columns`, `sheet_column`, per-source `header_row`, `reason`, + `ignored_columns`, `renamed_columns` and `duplicates_removed`, and totals + `tables_merged`, `duplicates_removed` and `excluded`. A bound mapping adds `mapping`. +- Limitations, in application-owned words, note left-out files, blank-filled columns and + left-out columns. The result stays complete: exclusion is a requested choice and is + disclosed rather than hidden. + +### Failure codes + +| Code | When | +| --- | --- | +| `merge_nothing_to_merge` | Every file or sheet was skipped or left out. | +| `merge_columns_not_found` | A duplicate key or sort column isn't in the merged table. | +| `merge_options_invalid` | Settings that can't be combined reached the engine. | +| `merge_mapping_invalid` | A bound mapping fails the profile at merge time. | +| `inspect_sources_invalid`, `inspect_limit_exceeded` | Inspection was asked for non-spreadsheet files, too many files, or more columns than fit the inspection size. | + +`merge_schema_mismatch` now suggests keeping every column, lining up names or leaving out +files. Engine messages, which can contain file and column names, are never shown as +failure text. + +## Usage + +Administrators need nothing new: **Inspect spreadsheets** and the new merge settings are +available wherever **Merge spreadsheets** is, and the Merge document action switch and +limits govern both. A narrowed orchestration Capabilities list must include **Inspect +spreadsheets** for the inspect recipes. + +Users ask in a V2 chat, for example: + +- "Merge these exports and keep every column." +- "Merge them and treat `cust_id` as `Customer ID`." +- "What columns do these files have? Then merge the ones that match into Excel." +- "Merge every sheet in these workbooks, remove duplicate order numbers keeping the + latest, and sort by date." + +See [Merge files in chat](../../guides/merge-files.md). + +## Testing and validation + +| Test | Covers | +| --- | --- | +| `functional_tests/test_tabular_merge_reconciliation.py` | Union, aliases and their validation, mapped columns, exclusion, nothing-to-merge, header rows, all-sheets mode, missing named sheets, duplicates (first, last, blank keys, union padding), sorting (numbers, dates, stability, bound), and inspection (compatibility, problems, title rows, size budget). | +| `functional_tests/test_orchestration_tabular_reconciliation.py` | The inspect descriptor and gating, plan rules for every option combination and the mapping producer, inspection without a model, inspect → compose mapping → merge through the real executor, the compose retry for an invalid mapping, nullable union columns rendered as blank CSV cells, exclusion summaries, failure codes, the profile validator and the planner recipe. | +| `functional_tests/test_v2_orchestration_merge_arguments.mjs` | Plan-card wording for every merge and inspection setting, defaults left out, refused values not worded, names kept as data. | +| `ui_tests/test_v2_orchestration_merge_arguments.py` | The real plan card and review in Chromium, light and narrow dark layouts, with a hostile column name. | +| `functional_tests/test_orchestration_workflow_*_off_golden.py` | Planner payload goldens regenerated and audited for the new descriptor, inputs, profile, recipe and guidance. | + +### Performance + +Merging stays single-pass. Union and mapped policies add no extra pass over the files; +rows written before a later column appears are padded on read-back. Duplicate removal +holds one 16-byte digest per distinct row; sorting holds one key tuple and one offset per +row up to 250,000 rows. Inspection reads every row once to count it. + +### Known limitations + +- Aliases apply to every file; a header that means different things in different files + can't be mapped per file. +- Sorting understands ISO dates only, and numbers without locale-specific decimal commas. +- Excluding a file is all or nothing; individual rows are never dropped for not fitting. +- Merge still appends rows; key-based joins remain out of scope. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 57208e8ad..c8ebfa4c7 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,29 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.219)** + +#### New Features + +* **Merge Spreadsheets Whose Columns Differ** + * V2 chat merges can now reconcile files whose columns don't match. A merge can keep every column from every file (a file without a column leaves it blank), keep only a listed set of columns, treat differently named headers as one column (for example `cust_id` and `CustomerID` as `Customer ID`), or leave out and report the files or sheets that don't fit instead of failing. + * Merges can also read headers below a title row, merge every visible sheet of each workbook with a **Source Sheet** column, remove duplicate rows (identical rows, or rows with the same key columns, keeping the first or last copy), and sort by up to three columns as text, numbers or dates (up to 250,000 rows). + * When the user asks chat to line the columns up, the plan inspects the files, a **Prepare content** step prepares a column mapping with the new `tabular_column_mapping_v1` profile, and the merge validates that mapping again before code applies it. No model copies or rewrites rows, and the merge report lists mapped, left-out and low-confidence columns. + * Merged columns that some files lack are retained as nullable and render as blank cells. A merge that leaves out files says so in its limitations and step summary. + * New fixed failure messages cover settings that can't be combined, an invalid prepared mapping, missing duplicate or sort columns, and merges where nothing fits; the "different columns" message now suggests the new options. + * (Ref: #1619, `functions_tabular_merge.py`, `functions_orchestration_merge.py`, `functions_orchestration_adapters.py`, `functions_orchestration_registry.py`, `functions_orchestration_schema.py`, `functions_orchestration_services.py`, [Phase 2](features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md), [Merge files in chat](../guides/merge-files.md)) + +* **Inspect Spreadsheets In V2 Chat** + * A new Gather capability, **Inspect spreadsheets** (`tabular_inspect`), reads the sheet names, column headers, row counts and a few sample rows of CSV or Excel files, and reports which files share the same columns, which columns look like the same thing under different names, likely title rows, and a suggested merge policy. A file that can't be read is reported instead of stopping the step, and no model is used. + * It answers questions such as "what columns do these files have?" and prepares merges whose columns differ. The **Enable Merge** switch and its chat file limit govern it, and it appears in the orchestration Capabilities list. + * (Ref: #1619, `run_tabular_inspect`, `inspect_tabular_sources`, `admin_settings_fields.py`, [Phase 2](features/V2_FILE_MERGE_PHASE_2_RECONCILIATION.md)) + +#### User Interface Enhancements + +* **Merge Settings In Words On The Plan Review** + * The V2 plan review now states merge and inspection settings in plain words, such as "Keep every column from every file" or "Remove rows with the same Customer ID, keeping the last", leaves out values that only repeat a default, and shows column and sheet names as plain text. + * (Ref: `application/v2_ui/src/lib/orchestrationMerge.ts`, `OrchestrationRunView.tsx`) + ### **(v0.261.218)** #### New Features diff --git a/docs/guides/index.md b/docs/guides/index.md index 06c549cea..59c705d47 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -50,7 +50,7 @@ Use this section when you want momentum first: each page is a practical path thr | [Use web search]({{ '/guides/use-web-search/' | relative_url }}) | Ground an answer in current public web results, and see exactly what leaves SimpleChat when you do. | Users | | [Review pasted URLs]({{ '/guides/review-pasted-urls/' | relative_url }}) | Use URL Access to review links included in the current chat message. | Users | | [Generate images]({{ '/guides/generate-images/' | relative_url }}) | Use the chat Image control to request AI-generated images. | Users | -| [Merge files in chat]({{ '/guides/merge-files/' | relative_url }}) | Combine several CSV or Excel files that share the same columns into one CSV or Excel file. | Users | +| [Merge files in chat]({{ '/guides/merge-files/' | relative_url }}) | Combine several CSV or Excel files into one CSV or Excel file, even when their columns differ. | Users | ## Documents and workspaces diff --git a/docs/guides/merge-files.md b/docs/guides/merge-files.md index 8c0910df6..256f753c0 100644 --- a/docs/guides/merge-files.md +++ b/docs/guides/merge-files.md @@ -1,23 +1,27 @@ --- layout: page title: "Merge files in chat" -description: "Combine several CSV or Excel files that share the same columns into one CSV or Excel file." +description: "Combine several CSV or Excel files into one CSV or Excel file, even when their columns differ." section: "Guides" audience: user -version: "0.261.218" +version: "0.261.219" --- ## What this does -Merging takes several spreadsheets with the same columns, such as one sales export per -region or one timesheet per month, and stacks their rows into a single table. The chat -then gives you that table as a downloadable CSV or Excel file. +Merging takes several spreadsheets, such as one sales export per region or one timesheet +per month, and stacks their rows into a single table. The chat then gives you that table +as a downloadable CSV or Excel file. Code does the merge, not a model. Every row is copied exactly, in the order you list the files, so codes such as `007` keep their leading zeros, and nothing is summarized, rounded or rewritten. The number of rows in the result is checked against the files it came from. +The files don't need identical columns. You can keep every column from every file, line +up columns that have different names, or leave out files that don't fit, and chat can +inspect the files first to show you how they differ. + ## Why you would use this Use it when you would otherwise open each file and copy-paste rows into one sheet, or ask @@ -31,11 +35,11 @@ for that work separately. ## Before you start - Use the V2 chat with **Orchestrate** on. An administrator must enable Chat - Orchestration and leave **Merge spreadsheets** and **Enable Merge** on; both are on by - default. See [orchestration settings]({{ '/admin/orchestration/' | relative_url }}). + Orchestration and leave **Merge spreadsheets**, **Inspect spreadsheets** and **Enable + Merge** on; all are on by default. See + [orchestration settings]({{ '/admin/orchestration/' | relative_url }}). - The files must be CSV (`.csv`) or Excel (`.xlsx`, `.xlsm`, `.xls`) documents you can open in SimpleChat, in a personal, group or public workspace. -- Every file needs the same column headers. The order of the columns can differ. - Chat merges up to 10 files and 250,000 rows by default. Your administrator may set different limits. @@ -51,34 +55,91 @@ for that work separately. > Stack the "Data" sheet from each of these workbooks into one CSV. 3. Review the plan. It shows a **Merge spreadsheets** task followed by the file to - create. Check the files listed, then run the plan. + create. Check the files and settings listed, then run the plan. 4. Download the file from the answer. If you don't select the files, you can describe them, for example "merge the monthly timesheet files from my workspace". The plan then searches for matching files first and merges the ones it finds, so check the plan's search wording before you run it. +## Check the files first + +Ask what the files contain before you merge them, for example: + +> What columns do these spreadsheets have, and do they match? + +The plan runs **Inspect spreadsheets**, which reads each file's sheet names, column +headers, row count and a few sample rows. The answer tells you which files share the same +columns, which columns look like the same thing under different names (such as +`Customer ID` and `customer_id`), and whether a file seems to have a title row above its +headers. Inspection never changes your files, and a file that can't be read is reported +rather than stopping the others. + +## When the columns differ + +Say how you want the differences handled. The plan card shows the choice in words before +you run it. + +| What you want | Ask for it like this | +| --- | --- | +| Keep every column from every file. A file that lacks a column leaves it blank. | "Merge these files and keep all the columns." | +| Treat differently named columns as one column. | "Merge them, and treat `cust_id` and `CustomerID` as `Customer ID`." | +| Let chat work out how the columns line up. | "Merge these files and line up the columns that mean the same thing." | +| Keep only some columns. | "Merge them with just the Customer ID, Name and Amount columns." | +| Leave out files or sheets that don't match. | "Merge the files that have the same columns as the first one and skip the rest." | + +When chat lines up the columns for you, it inspects the files, prepares a column mapping +and then merges with it. Code checks the mapping before any row is copied, and the merge +report lists which headers were mapped, which were left out, and any mapping chat marked +as uncertain. To review the differences before anything is merged, ask chat to inspect +the files first and merge in your next message. + +Files that are left out are named in the merge report, and the merge step's summary says +how many were left out. The merged rows are still complete for the files that were merged. + +## More ways to shape the merge + +- **Headers below a title.** If each file has a title or notes above the headers, say + which row holds the headers: "the headers are on row 3". +- **Every sheet.** Ask to merge every sheet of each workbook. Hidden and empty sheets are + skipped, and a **Source Sheet** column records each row's sheet. +- **Duplicates.** Ask to remove rows that are exactly the same, or rows with the same + value in one or more columns, such as the same order number. You can keep the first or + the last copy. Missing values and empty values count as the same, and rows whose key is + blank are always kept. +- **Order.** Ask to sort by up to three columns, as text, numbers or dates. Numbers may + use thousands separators or a leading currency symbol; dates sort when they're written + as `YYYY-MM-DD`, which is how Excel dates are copied. Values that aren't numbers or + dates sort after those that are, and blanks come last. Sorting is available for merges + of up to 250,000 rows. + ## What you get - **Columns** follow the first file's column order and spelling. Headers in the other - files are matched by name, ignoring capitalization and extra spaces. + files are matched by name, ignoring capitalization and extra spaces. When you keep + every column, columns found only in later files are added at the end. - **Source File** is added as the first column unless you ask not to include it. If your files already have a column with that name, the new column is called **Source File (2)** instead. -- **Rows** appear file by file, in the order the files were listed. Completely blank - rows are skipped, and rows with fewer values than columns are padded with blanks. +- **Rows** appear file by file, in the order the files were listed, unless you ask for a + sort. Completely blank rows are skipped, and rows with fewer values than columns are + padded with blanks. - **Excel values** are copied as plain text. Dates become `YYYY-MM-DD`, true and false become `TRUE` and `FALSE`, and formulas contribute their last calculated value. -- **Workbooks** contribute their first visible sheet unless you name a sheet, and the - same sheet name is used in every workbook. +- **Workbooks** contribute their first visible sheet unless you name a sheet or ask for + every sheet. ## If the merge doesn't run | Message | What to do | | --- | --- | -| The selected files don't all have the same columns | Check that every file has the same headers. Remove the files that differ, or fix their headers, and ask again. | +| The selected files don't all have the same columns | Ask to keep every column, to line up columns that have different names, or to leave out the files that don't match. | | A selected file couldn't be read as a CSV or Excel file | The file may be damaged or password-protected. Open and save it again without a password. | -| A selected workbook doesn't have the requested sheet | Check the sheet name, or leave it out to use each workbook's first visible sheet. | +| A selected workbook doesn't have the requested sheet | Check the sheet name, leave it out to use each workbook's first visible sheet, or ask to skip workbooks without it. | +| A column named for removing duplicates or sorting isn't in the merged files | Check the column's spelling against the files' headers. Inspecting the files first lists them. | +| None of the selected files had a sheet with column headers that fit the merge | Check that the files have headers, or say which row holds them. | +| This merge's settings can't be used together | Ask again with fewer instructions at once, for example keep every column or list the columns to keep, not both. | +| The column mapping prepared for this merge isn't valid | Ask again, or name the columns to treat as the same yourself. | | The merge is larger than chat allows | Merge fewer or smaller files at a time, or ask your administrator about the Merge limits. | | Merging needs at least two different CSV or Excel files | Select at least two spreadsheets. Word, PDF and other documents can't be merged as tables. | diff --git a/functional_tests/test_orchestration_dependency_allowlist.py b/functional_tests/test_orchestration_dependency_allowlist.py index c84e261f3..8df697147 100644 --- a/functional_tests/test_orchestration_dependency_allowlist.py +++ b/functional_tests/test_orchestration_dependency_allowlist.py @@ -1,10 +1,11 @@ # test_orchestration_dependency_allowlist.py """Functional tests for authoritative capability allowlists and admin metadata. -Version: 0.261.207 +Version: 0.261.219 Implemented in: 0.261.127 Single orchestration contract updated in: 0.261.139 Workflow proposal capability described in: 0.261.207 +Spreadsheet inspection and merge capabilities listed in: 0.261.219 A capability list saved before Gather / Reason / Render became the only contract keeps its answering ability, because a stored `respond` reads as `compose`, and never silently opts into file publication. The stored list itself is never rewritten. @@ -208,6 +209,7 @@ def test_registry_metadata_expresses_roles_without_phases_or_a_terminal_step(run 'contract_version': 2, 'capability_ids': [ 'document_search', 'document_analyze', 'document_compare', 'tabular_analyze', + 'tabular_inspect', 'tabular_merge', 'web_search', 'url_fetch', 'deep_research', 'action_invoke', 'agent_invoke', 'compose', 'generate_image', 'workflow_propose', 'workflow_run', 'workflow_results', 'render_file', ], diff --git a/functional_tests/test_orchestration_tabular_merge_capability.py b/functional_tests/test_orchestration_tabular_merge_capability.py index e62333b43..7cf3e48d8 100644 --- a/functional_tests/test_orchestration_tabular_merge_capability.py +++ b/functional_tests/test_orchestration_tabular_merge_capability.py @@ -72,7 +72,7 @@ def test_tabular_merge_is_a_gated_reason_capability(runtime): assert capability['label'] == 'Merge spreadsheets' assert capability['document_action_type'] == 'merge' assert capability['result_outputs'] == {'records': 'records-v1', 'report': 'structured-v1'} - assert capability['result_input_kinds'] == {'sources': ('source-set-v1',)} + assert capability['result_input_kinds'] == {'sources': ('source-set-v1',), 'mapping': ('structured-v1',)} assert capability['inputs']['properties']['document_ids']['minItems'] == 2 assert capability['adapter'] == 'tabular_merge' assert 'tabular_merge' in runtime.contracts.REASON_CAPABILITIES @@ -122,7 +122,7 @@ def test_plans_name_sources_explicitly_or_bind_a_source_set(runtime): with pytest.raises(schema.PlanValidationError): runtime.make([merge_step(['east-doc'])], available=AVAILABLE) with pytest.raises(schema.PlanValidationError): - runtime.make([merge_step(['east-doc', 'west-doc'], schema_policy='union')], available=AVAILABLE) + runtime.make([merge_step(['east-doc', 'west-doc'], schema_policy='crosswalk')], available=AVAILABLE) def test_the_chat_document_limit_and_source_kinds_are_enforced_at_planning(runtime): @@ -186,7 +186,10 @@ def test_executor_retains_exact_rows_and_a_report_without_a_model(runtime): assert records.completeness.coverage.unit == 'sources' report = service.open_result(task.output('report')).read_value() assert report['status'] == 'merged' - assert report['totals'] == {'sources': 2, 'rows': 3, 'columns': 4, 'blank_rows_skipped': 0} + assert report['totals'] == { + 'sources': 2, 'tables_merged': 2, 'rows': 3, 'columns': 4, 'blank_rows_skipped': 0, + 'duplicates_removed': 0, 'excluded': 0, + } assert [entry['file_name'] for entry in report['sources']] == ['east.csv', 'west.csv'] metadata = service.open_result(task.output('records')).metadata() assert metadata['origin'] == 'grounded' diff --git a/functional_tests/test_orchestration_tabular_reconciliation.py b/functional_tests/test_orchestration_tabular_reconciliation.py new file mode 100644 index 000000000..fa7c3f564 --- /dev/null +++ b/functional_tests/test_orchestration_tabular_reconciliation.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +# test_orchestration_tabular_reconciliation.py +""" +Functional test for reconciling differently structured spreadsheets in chat orchestration. +Version: 0.261.219 +Implemented in: 0.261.219 + +This test ensures that tabular_inspect is a gated Gather capability that retains a bounded +inspection without a model, that tabular_merge plans accept union, aliases, exclusion, +de-duplication, sorting and a bound column mapping only in valid combinations, that a +mapping must come from a compose step using the tabular_column_mapping_v1 profile, that the +real executor runs inspect -> compose mapping -> merge end to end with the mapping +re-validated at merge time, that union merges retain nullable columns that render as blank +cells, and that the planner is offered the reconciliation recipe. +""" + +import csv +import io +import json +import sys + +import pytest + +from test_orchestration_dependency_runtime import binding, execute, runtime # noqa: F401 +from test_support.versioning import assert_app_version_at_least + + +PROFILE = 'tabular_column_mapping_v1' +AVAILABLE = ['tabular_inspect', 'tabular_merge', 'document_search', 'compose', 'render_file'] +FILES = { + 'crm-doc': ('crm.csv', b'Customer ID,Name,Amount\r\n001,Ann,10\r\n002,Bob,5\r\n'), + 'erp-doc': ('erp.csv', b'cust_id,Full Name,Total,Notes\n003,Cy,7,vip\n'), + 'web-doc': ('web.csv', b'Name,Channel\nDee,Online\n'), + 'notes-doc': ('notes.docx', b'not a spreadsheet'), +} +MAPPING = { + 'columns': ['Customer ID', 'Name', 'Amount'], + 'mappings': [ + {'source_column': 'cust_id', 'target_column': 'Customer ID', 'confidence': 'high'}, + {'source_column': 'Full Name', 'target_column': 'Name', 'confidence': 'medium'}, + {'source_column': 'Total', 'target_column': 'Amount', 'confidence': 'low'}, + {'source_column': 'Notes', 'target_column': None, 'confidence': 'high'}, + ], + 'notes': 'Lined up the customer columns.', +} + + +def step(step_id, capability_id, document_ids=None, **arguments): + value = {'step_id': step_id, 'capability_id': capability_id, 'arguments': dict(arguments)} + if document_ids is not None: + value['arguments']['document_ids'] = list(document_ids) + return value + + +def mapping_compose(inputs=None, outputs=None): + return { + 'step_id': 'map', 'capability_id': 'compose', + 'arguments': {'instruction': 'Line up the columns of these files for one merged table.'}, + 'inputs': inputs if inputs is not None else { + 'inspection': {'binding': binding('inspect', 'inspection'), 'allow_partial': False}, + }, + 'outputs': outputs or [{'name': 'mapping', 'kind': 'structured-v1', 'profile': PROFILE}], + } + + +def mapped_merge(document_ids=('crm-doc', 'erp-doc'), **arguments): + value = step('merge', 'tabular_merge', document_ids, **arguments) + value['inputs'] = {'mapping': {'binding': binding('map', 'mapping'), 'allow_partial': False}} + return value + + +def profiles(): + from functions_orchestration_services import composition_profiles, validate_composition_profile + + return composition_profiles(), validate_composition_profile + + +def tabular_case(runtime, steps, replies=(), *, settings=None, available=AVAILABLE): + profile_map, validator = profiles() + case = runtime.make( + steps, replies, settings=settings, available=available, profiles=profile_map, + profile_validator=validator, + ) + for document_id, (file_name, _) in FILES.items(): + case.fixture.sources[document_id] = { + 'document_id': document_id, 'scope': 'personal', 'scope_id': 'owner', + 'source_version': 1, 'source_revision': f'{document_id}-revision', + 'file_name': file_name, + 'source_kind': 'tabular' if file_name.endswith('.csv') else 'narrative', + } + reads = [] + + def reader(source, user_id, group_id=None, public_workspace_id=None, *, purpose): + assert user_id == 'owner' and purpose == 'native' + reads.append(source['document_id']) + return {'id': source['document_id']}, FILES[source['document_id']][1] + + case.context.merge_source_reader = reader + case.reads = reads + return case + + +def make_plan(runtime, steps, **kwargs): + profile_map, _ = profiles() + return runtime.make(steps, available=kwargs.pop('available', AVAILABLE), profiles=profile_map, **kwargs).plan + + +def test_version_includes_reconciliation(): + assert_app_version_at_least('0.261.219') + + +def test_tabular_inspect_is_a_gated_gather_capability(runtime): + registry = runtime.registry + capability = registry.get_capability('tabular_inspect') + assert capability['role'] == registry.ROLE_GATHER + assert capability['label'] == 'Inspect spreadsheets' + assert capability['result_outputs'] == {'inspection': 'structured-v1'} + assert capability['result_input_kinds'] == {'sources': ('source-set-v1',)} + assert capability['document_action_type'] == 'merge' + assert capability['inputs']['properties']['document_ids']['minItems'] == 1 + assert 'tabular_inspect' not in runtime.contracts.REASON_CAPABILITIES + + settings = {'enable_chat_orchestration': True, 'enable_user_workspace': True} + available = {item['id'] for item in registry.resolve_available_capabilities(settings)} + assert {'tabular_inspect', 'tabular_merge'} <= available + disabled = {**settings, 'document_action_capabilities': {'merge': {'enabled': False}}} + available = {item['id'] for item in registry.resolve_available_capabilities(disabled)} + assert not {'tabular_inspect', 'tabular_merge'} & available + + from admin_settings_fields import ADMIN_SETTINGS_FIELDS + + options = [ + option['value'] for fields in ADMIN_SETTINGS_FIELDS.values() for field in fields + if field.get('key') == 'chat_orchestration_enabled_capabilities' for option in field['options'] + ] + assert options.index('tabular_inspect') < options.index('tabular_merge') + + +def test_inspect_plans_name_sources_or_bind_a_source_set(runtime): + schema = runtime.schema + plan = make_plan(runtime, [step('inspect', 'tabular_inspect', ['crm-doc'])]) + assert plan['steps'][0]['arguments'] == {'document_ids': ['crm-doc'], 'doc_scope': 'all'} + with pytest.raises(schema.PlanValidationError) as missing: + make_plan(runtime, [step('inspect', 'tabular_inspect')]) + assert missing.value.code == 'source_binding_required' + with pytest.raises(schema.PlanValidationError): + make_plan(runtime, [step('inspect', 'tabular_inspect', ['crm-doc'], sheet='Data', sheets='all')]) + with pytest.raises(schema.PlanValidationError): + make_plan(runtime, [step('inspect', 'tabular_inspect', ['crm-doc'], sample_rows=11)]) + + plan = make_plan(runtime, [step('inspect', 'tabular_inspect', ['crm-doc', 'notes-doc'])]) + with pytest.raises(schema.PlanValidationError) as kinds: + schema.validate_plan_document_source_kinds(plan, {'crm-doc': 'tabular', 'notes-doc': 'narrative'}) + assert kinds.value.code == 'source_kind_invalid' + + +@pytest.mark.parametrize('arguments', [ + {'schema_policy': 'union', 'on_incompatible': 'exclude'}, + {'column_aliases': {'Customer ID': ['cust_id']}, 'header_row': 2}, + {'schema_policy': 'mapped', 'columns': ['Customer ID', 'Amount']}, + {'sheets': 'all', 'dedupe': 'key_columns', 'dedupe_columns': ['Customer ID'], 'dedupe_keep': 'last'}, + {'dedupe': 'exact_rows', 'sort_by': [{'column': 'Amount', 'descending': True, 'value_type': 'number'}]}, +]) +def test_reconciliation_arguments_validate(runtime, arguments): + plan = make_plan(runtime, [step('merge', 'tabular_merge', ['crm-doc', 'erp-doc'], **arguments)]) + for key, value in arguments.items(): + assert plan['steps'][0]['arguments'][key] == value + + +@pytest.mark.parametrize('arguments', [ + {'columns': ['Customer ID']}, + {'schema_policy': 'mapped'}, + {'sheet': 'Data', 'sheets': 'all'}, + {'dedupe_columns': ['Customer ID']}, + {'dedupe': 'key_columns'}, + {'column_aliases': {'Customer ID': ['Amount'], 'Amount': ['Total']}}, + {'sort_by': [{'column': 'A'}, {'column': 'a'}]}, +]) +def test_conflicting_merge_settings_are_refused_at_planning(runtime, arguments): + with pytest.raises(runtime.schema.PlanValidationError) as refused: + make_plan(runtime, [step('merge', 'tabular_merge', ['crm-doc', 'erp-doc'], **arguments)]) + assert refused.value.code in ('merge_options_invalid', 'plan_invalid') + + +def test_a_bound_mapping_must_be_a_prepared_column_mapping(runtime): + schema = runtime.schema + inspect = step('inspect', 'tabular_inspect', ['crm-doc', 'erp-doc']) + plan = make_plan(runtime, [inspect, mapping_compose(), mapped_merge()]) + merge = next(item for item in plan['steps'] if item['step_id'] == 'merge') + assert merge['depends_on'] == ['map'] + + with pytest.raises(schema.PlanValidationError) as unprofiled: + make_plan(runtime, [inspect, mapping_compose(outputs=[{'name': 'mapping', 'kind': 'structured-v1'}]), mapped_merge()]) + assert unprofiled.value.code == 'mapping_profile_required' + for arguments in ({'columns': ['Customer ID']}, {'column_aliases': {'Name': ['Full Name']}}, {'schema_policy': 'union'}): + with pytest.raises(schema.PlanValidationError) as conflict: + make_plan(runtime, [inspect, mapping_compose(), mapped_merge(**arguments)]) + assert conflict.value.code == 'merge_options_invalid' + make_plan(runtime, [inspect, mapping_compose(), mapped_merge(schema_policy='mapped', dedupe='exact_rows')]) + + +def test_inspection_is_retained_without_a_model(runtime): + case = tabular_case(runtime, [step('inspect', 'tabular_inspect', ['crm-doc', 'erp-doc'], sample_rows=1)]) + result = execute(runtime, case) + assert result['status'] == 'completed', result + assert case.model.calls == [] + assert case.reads == ['crm-doc', 'erp-doc'] + task = runtime.contracts.TaskResult.from_dict(result['task_results']['inspect']) + assert task.status == 'complete' + reference = task.output('inspection') + assert reference.completeness.coverage.unit == 'sources' + inspection = case.fixture.restart().open_result(reference).read_value() + assert inspection['version'] == 'tabular-inspection-v1' + assert [entry['file_name'] for entry in inspection['sources']] == ['crm.csv', 'erp.csv'] + assert inspection['sources'][1]['tables'][0]['columns'] == ['cust_id', 'Full Name', 'Total', 'Notes'] + assert inspection['compatibility']['identical_columns'] is False + assert inspection['compatibility']['suggested_policy'] == 'union' + + +def test_inspect_compose_mapping_merge_runs_end_to_end(runtime): + steps = [step('inspect', 'tabular_inspect', ['crm-doc', 'erp-doc']), mapping_compose(), mapped_merge()] + case = tabular_case(runtime, steps, [json.dumps({'mapping': MAPPING})]) + result = execute(runtime, case) + assert result['status'] == 'completed', result + assert len(case.model.calls) == 1 + payload = json.loads(case.model.calls[0][0][-1]['content']) + assert payload['profiles'][PROFILE]['required'] == ['columns', 'mappings'] + assert payload['inputs']['inspection']['value']['version'] == 'tabular-inspection-v1' + + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + service = case.fixture.restart() + rows = list(service.open_result(task.output('records')).iter_records()) + assert rows == [ + {'Source File': 'crm.csv', 'Customer ID': '001', 'Name': 'Ann', 'Amount': '10'}, + {'Source File': 'crm.csv', 'Customer ID': '002', 'Name': 'Bob', 'Amount': '5'}, + {'Source File': 'erp.csv', 'Customer ID': '003', 'Name': 'Cy', 'Amount': '7'}, + ] + report = service.open_result(task.output('report')).read_value() + assert report['policy']['schema_policy'] == 'mapped' + assert report['policy']['column_aliases'] == { + 'Customer ID': ['cust_id'], 'Name': ['Full Name'], 'Amount': ['Total'], + } + assert report['mapping'] == { + 'profile': PROFILE, 'explicitly_ignored': ['Notes'], + 'low_confidence': [{'source_column': 'Total', 'target_column': 'Amount'}], + 'notes': 'Lined up the customer columns.', + } + assert report['sources'][1]['ignored_columns'] == ['Notes'] + + +def test_an_invalid_prepared_mapping_is_asked_for_once_more_then_fails(runtime): + invalid = {**MAPPING, 'mappings': [{'source_column': 'Total', 'target_column': 'Revenue', 'confidence': 'high'}]} + steps = [step('inspect', 'tabular_inspect', ['crm-doc', 'erp-doc']), mapping_compose(), mapped_merge()] + case = tabular_case(runtime, steps, [json.dumps({'mapping': invalid}), json.dumps({'mapping': invalid})]) + result = execute(runtime, case) + assert result['status'] == 'failed' + assert len(case.model.calls) == 2 + assert 'merge' not in result['task_results'] + assert case.reads == ['crm-doc', 'erp-doc'] + + +def test_union_merge_retains_nullable_columns_that_render_blank(runtime): + from functions_generated_file_exports import GeneratedFileExportRequest, build_generated_file_export + from functions_orchestration_export_sources import open_orchestration_export_source + + case = tabular_case(runtime, [step( + 'merge', 'tabular_merge', ['crm-doc', 'web-doc'], schema_policy='union', + sort_by=[{'column': 'Name', 'descending': True}], + )]) + result = execute(runtime, case) + assert result['status'] == 'completed', result + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + records = task.output('records') + assert [(column.name, column.nullable) for column in records.columns] == [ + ('Source File', False), ('Customer ID', True), ('Name', False), ('Amount', True), ('Channel', True), + ] + assert any('left blank' in text for text in records.completeness.limitations) + source = open_orchestration_export_source(case.fixture.restart(), records) + with build_generated_file_export( + source=source, export_request=GeneratedFileExportRequest('csv', 'exact_tabular_records_v1'), + max_output_bytes=1024 * 1024, + ) as output: + content = output.file_content.read() + source.require_complete_consumption() + assert list(csv.reader(io.StringIO(content.decode('utf-8')))) == [ + ['Source File', 'Customer ID', 'Name', 'Amount', 'Channel'], + ['web.csv', '', 'Dee', '', 'Online'], + ['crm.csv', '002', 'Bob', '5', ''], + ['crm.csv', '001', 'Ann', '10', ''], + ] + + +def test_excluded_files_are_reported_in_the_summary_and_limitations(runtime): + case = tabular_case(runtime, [step( + 'merge', 'tabular_merge', ['crm-doc', 'web-doc'], on_incompatible='exclude', + )]) + result = execute(runtime, case) + assert result['status'] == 'completed', result + task = runtime.contracts.TaskResult.from_dict(result['task_results']['merge']) + records = task.output('records') + assert records.item_count == 2 + assert any('were left out' in text for text in records.completeness.limitations) + summary = next(item['summary'] for item in result['steps'] if item['step_id'] == 'merge') + assert summary == ( + "Merged 2 row(s) from 2 file(s) into 4 column(s); left out 1 file(s) or sheet(s) whose columns don't fit." + ) + + +def test_missing_dedupe_columns_use_an_application_owned_failure(runtime): + case = tabular_case(runtime, [step( + 'merge', 'tabular_merge', ['crm-doc', 'web-doc'], schema_policy='union', + dedupe='key_columns', dedupe_columns=['Order ID'], + )]) + result = execute(runtime, case) + assert result['status'] == 'failed' + assert result['failure']['code'] == 'merge_columns_not_found' + assert 'Order ID' not in result['message'] + + +def test_composition_profile_validator_accepts_only_valid_mappings(): + from functions_orchestration_result_contracts import ResultContractError + from functions_orchestration_services import composition_profiles, validate_composition_profile + + assert set(composition_profiles()) == {'prepared_slide_deck_v1', PROFILE} + assert validate_composition_profile(PROFILE, MAPPING) is True + for invalid in ( + {'columns': ['A'], 'mappings': [], 'extra': True}, + {'columns': ['A', 'a'], 'mappings': []}, + {'columns': ['A'], 'mappings': [{'source_column': 'B', 'target_column': 'C', 'confidence': 'high'}]}, + {'columns': ['A', 'B'], 'mappings': [{'source_column': 'B', 'target_column': 'A', 'confidence': 'high'}]}, + {'columns': ['A'], 'mappings': [{'source_column': 'A', 'target_column': None, 'confidence': 'high'}]}, + {'columns': ['A'], 'mappings': [{'source_column': 'x', 'target_column': 'A', 'confidence': 'sure'}]}, + {'columns': ['A'], 'mappings': [ + {'source_column': 'x', 'target_column': 'A', 'confidence': 'high'}, + {'source_column': 'X', 'target_column': None, 'confidence': 'high'}, + ]}, + {'columns': [], 'mappings': []}, + ): + with pytest.raises(ResultContractError) as refused: + validate_composition_profile(PROFILE, invalid) + assert refused.value.code == 'result_schema_invalid' + + +def test_planner_is_offered_the_reconciliation_recipe(runtime): + from functions_orchestration_deliverables import build_deliverable_availability + + settings = {'enable_chat_orchestration': True, 'enable_user_workspace': True} + capabilities = [ + runtime.registry.get_capability(capability_id) + for capability_id in ('tabular_inspect', 'tabular_merge', 'compose', 'render_file') + ] + availability = build_deliverable_availability( + settings, capabilities=capabilities, unavailable={}, export_catalog=None, + ) + recipes = {recipe['for']: recipe['steps'] for recipe in availability['recipes']} + steps = recipes['One file merged from CSV or Excel files whose columns differ'] + assert PROFILE in steps and 'tabular_inspect' in steps + + without_inspect = build_deliverable_availability( + settings, capabilities=[item for item in capabilities if item['id'] != 'tabular_inspect'], + unavailable={}, export_catalog=None, + ) + assert 'One file merged from CSV or Excel files whose columns differ' not in { + recipe['for'] for recipe in without_inspect['recipes'] + } + + +if __name__ == '__main__': + sys.exit(pytest.main([__file__, '-q'])) diff --git a/functional_tests/test_support/orchestration_workflow_results_off_golden.json b/functional_tests/test_support/orchestration_workflow_results_off_golden.json index b3a749dc9..66f4cd4b6 100644 --- a/functional_tests/test_support/orchestration_workflow_results_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_results_off_golden.json @@ -1366,9 +1366,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -1678,9 +1679,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -1946,9 +1948,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -2583,12 +2586,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -2615,16 +2694,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -2636,6 +2759,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -2650,6 +2840,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -3313,6 +3506,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -3321,6 +3578,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -4283,12 +4541,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -4315,16 +4649,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -4336,6 +4714,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -4350,6 +4795,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -4958,6 +5406,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -4966,6 +5478,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -5864,12 +6377,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -5896,16 +6485,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -5917,6 +6550,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -5931,6 +6631,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -6503,6 +7206,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -6511,6 +7278,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -7239,12 +8007,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -7271,16 +8115,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -7292,6 +8180,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -7306,6 +8261,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -7914,6 +8872,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -7922,6 +8944,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8527,6 +9550,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8548,6 +9572,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8569,6 +9594,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8590,6 +9616,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8611,6 +9638,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -8656,11 +9684,18 @@ "summary": "Compute over CSV or Excel data rather than reading it as prose.", "cost": "medium" }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "cost": "low" + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", "cost": "medium" }, { @@ -8742,11 +9777,18 @@ "summary": "Compute over CSV or Excel data rather than reading it as prose.", "cost": "medium" }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "cost": "low" + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", "cost": "medium" }, { diff --git a/functional_tests/test_support/orchestration_workflow_runs_off_golden.json b/functional_tests/test_support/orchestration_workflow_runs_off_golden.json index 917155ed6..37cc10271 100644 --- a/functional_tests/test_support/orchestration_workflow_runs_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_runs_off_golden.json @@ -665,9 +665,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -954,9 +955,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -1591,12 +1593,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -1623,16 +1701,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -1644,6 +1766,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -1658,6 +1847,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -2285,6 +2477,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -2293,6 +2549,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -3237,12 +3494,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -3269,16 +3602,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -3290,6 +3667,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -3304,6 +3748,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -3876,6 +4323,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -3884,6 +4395,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -4215,6 +4727,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -4235,6 +4748,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -4255,6 +4769,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -4275,6 +4790,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", diff --git a/functional_tests/test_support/orchestration_workflow_setting_off_golden.json b/functional_tests/test_support/orchestration_workflow_setting_off_golden.json index 192296c7b..311ff13f5 100644 --- a/functional_tests/test_support/orchestration_workflow_setting_off_golden.json +++ b/functional_tests/test_support/orchestration_workflow_setting_off_golden.json @@ -82,9 +82,10 @@ "invent one. If the user already selected documents, plan around those documents.", "Use each candidate's server-resolved source_kind, not its display label, to choose compatible", "work. Never send tabular source IDs to document_analyze or document_compare: those steps", - "do not admit native tabular inputs. To combine the rows of several tabular sources that share", - "the same columns into one table or file, use tabular_merge with every one of those sources, never", - "compose; it appends rows and cannot match rows on a key, so ask when the user may mean that.", + "do not admit native tabular inputs. To combine the rows of several tabular sources into one", + "table or file, use tabular_merge with every one of those sources, never compose; it appends", + "rows and cannot match rows on a key, so ask when the user may mean that. When their columns", + "may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first.", "For mixed narrative/tabular comparisons, prepare each", "source with compatible offered capabilities and compose their named results. Ask each", "preparation step only for its own sources' contribution, such as values, periods, units and", @@ -715,12 +716,88 @@ } ] }, + { + "id": "tabular_inspect", + "label": "Inspect spreadsheets", + "role": "gather", + "summary": "Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, and how their columns line up.", + "when_to_use": "Use before tabular_merge when the files may not share the same columns, or when the user asks what sheets, columns or rows CSV or Excel files have. It reports each file's headers, row counts and sample rows, which files share the same columns, near matches such as \"Customer ID\" and \"customer_id\", a likely title row with the header_row to use, and a suggested schema_policy. A file that cannot be read is reported, not fatal. To line up differently named columns, bind its \"inspection\" output to a compose step that declares a structured-v1 output with profile tabular_column_mapping_v1, and bind that output as tabular_merge's \"mapping\" input. When the user should review the differences first, compose an answer from the inspection and merge in a later turn.", + "inputs": { + "type": "object", + "properties": { + "document_ids": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "The CSV or Excel files to inspect." + }, + "doc_scope": { + "type": "string", + "enum": [ + "all", + "personal", + "group", + "public" + ], + "default": "all" + }, + "sheet": { + "type": "string", + "minLength": 1, + "maxLength": 31, + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." + }, + "sample_rows": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "description": "Sample rows to read per sheet; 3 when omitted." + } + }, + "additionalProperties": false + }, + "cost": "low", + "produces": [ + "retained_results" + ], + "max_per_plan": 2, + "plan_contract_version": 2, + "result_contract_version": "tabular-inspect-v1", + "result_input_kinds": { + "sources": [ + "source-set-v1" + ] + }, + "partial_inputs_supported": false, + "result_outputs": { + "inspection": "structured-v1" + }, + "optional_result_outputs": {}, + "result_output_kinds": [] + }, { "id": "tabular_merge", "label": "Merge spreadsheets", "role": "reason", - "summary": "Combine the rows of two or more CSV or Excel files that share the same columns into one table, exactly and without a model.", - "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files with the same columns into one table or file. Every row is kept exactly, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. Columns are matched by name in any order; schema_policy exact_order also requires the same order. sheet picks one sheet in every workbook; otherwise each workbook's first visible sheet is read. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", + "summary": "Combine the rows of two or more CSV or Excel files into one table, exactly and without a model, lining up columns by name, alias or a prepared mapping.", + "when_to_use": "Use when the user asks to merge, combine, append, stack or consolidate several CSV or Excel files into one table or file. Every row is kept, in the order the files are listed. It never matches rows on a key column (no joins or lookups) and never edits values; when \"merge\" could mean matching rows on a key, ask. Name the files in document_ids in merge order, or bind inputs.sources to a search step's sources output when the files must be found first. schema_policy: by_name (default) needs the same columns in any order, exact_order also the same order, union keeps every column from every file and leaves a missing one blank, mapped keeps exactly \"columns\". column_aliases lines up other header names, e.g. {\"Customer ID\": [\"cust_id\"]}. When the columns may differ and how they line up is unclear, run tabular_inspect first and bind a compose step's tabular_column_mapping_v1 output as the \"mapping\" input, without columns or aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit instead of failing. header_row skips title rows above the headers. sheet picks one sheet; sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last (dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds each row's file name. To deliver a file, bind the records output to render_file with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose merged rows. A merge of more files than the chat limit, or one that should repeat, belongs in a workflow.", "inputs": { "type": "object", "properties": { @@ -747,16 +824,60 @@ "type": "string", "enum": [ "by_name", - "exact_order" + "exact_order", + "union", + "mapped" ], "default": "by_name", - "description": "by_name accepts the same columns in any order; exact_order also requires the same order." + "description": "by_name: same columns in any order. exact_order: same columns in the same order. union: every column from every file. mapped: exactly \"columns\"." + }, + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true, + "description": "mapped only: the merged columns, in order." + }, + "column_aliases": { + "type": "object", + "maxProperties": 256, + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 32, + "uniqueItems": true + }, + "description": "Each column name mapped to the other header names that mean it." }, "sheet": { "type": "string", "minLength": 1, "maxLength": 31, - "description": "The sheet to read in every Excel file. Omit for each workbook's first visible sheet." + "description": "One sheet to read in every Excel file. Omit for each workbook's first visible sheet." + }, + "sheets": { + "type": "string", + "enum": [ + "first", + "all" + ], + "description": "all reads every visible sheet of each workbook; first (default) reads one." + }, + "header_row": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "description": "The row holding the column headers in every file. Omit to use the first nonblank row." }, "include_source_column": { "type": "boolean", @@ -768,6 +889,73 @@ "minLength": 1, "maxLength": 128, "default": "Source File" + }, + "on_incompatible": { + "type": "string", + "enum": [ + "fail", + "exclude" + ], + "description": "fail (default) merges nothing when a file does not fit; exclude leaves it out." + }, + "dedupe": { + "type": "string", + "enum": [ + "none", + "exact_rows", + "key_columns" + ], + "description": "Remove duplicate rows; none when omitted." + }, + "dedupe_columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 16, + "uniqueItems": true, + "description": "key_columns only: the columns that identify a duplicate." + }, + "dedupe_keep": { + "type": "string", + "enum": [ + "first", + "last" + ], + "description": "Which duplicate to keep; first when omitted." + }, + "sort_by": { + "type": "array", + "maxItems": 3, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "descending": { + "type": "boolean" + }, + "value_type": { + "type": "string", + "enum": [ + "text", + "number", + "date" + ] + } + }, + "required": [ + "column" + ], + "additionalProperties": false + }, + "description": "Sort keys, most significant first; text ascending when not stated." } }, "additionalProperties": false @@ -782,6 +970,9 @@ "result_input_kinds": { "sources": [ "source-set-v1" + ], + "mapping": [ + "structured-v1" ] }, "partial_inputs_supported": false, @@ -1354,6 +1545,70 @@ } } } + }, + "tabular_column_mapping_v1": { + "type": "object", + "description": "How to line up the columns of several CSV or Excel files before merging them. columns is the merged table's columns in order; use the clearest existing header for each. mappings lists every header whose name differs from the column it fills, and every header to leave out (target_column null). A header that already matches a column, ignoring case and spacing, needs no entry. Map a header only when it holds the same kind of value; when unsure, keep it as its own column or mark confidence low.", + "properties": { + "columns": { + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 255, + "uniqueItems": true + }, + "mappings": { + "type": "array", + "maxItems": 1024, + "items": { + "type": "object", + "properties": { + "source_column": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "A header exactly as it appears in a file." + }, + "target_column": { + "type": [ + "string", + "null" + ], + "maxLength": 256, + "description": "The column in columns this header fills, or null to leave it out." + }, + "confidence": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + } + }, + "required": [ + "source_column", + "target_column", + "confidence" + ], + "additionalProperties": false + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "A short explanation of the mapping for the user." + } + }, + "required": [ + "columns", + "mappings" + ], + "additionalProperties": false } }, "capability_availability": { @@ -1362,6 +1617,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", @@ -1613,6 +1869,7 @@ "document_analyze", "document_compare", "tabular_analyze", + "tabular_inspect", "tabular_merge", "web_search", "action_invoke", diff --git a/functional_tests/test_tabular_merge_engine.py b/functional_tests/test_tabular_merge_engine.py index a96581c35..f496befba 100644 --- a/functional_tests/test_tabular_merge_engine.py +++ b/functional_tests/test_tabular_merge_engine.py @@ -84,7 +84,10 @@ def test_appends_rows_in_file_order_with_a_source_column(): report = result.report assert report["version"] == "tabular-merge-report-v1" assert report["status"] == "merged" - assert report["totals"] == {"sources": 2, "rows": 3, "columns": 4, "blank_rows_skipped": 0} + assert report["totals"] == { + "sources": 2, "tables_merged": 2, "rows": 3, "columns": 4, "blank_rows_skipped": 0, + "duplicates_removed": 0, "excluded": 0, + } assert [entry["rows"] for entry in report["sources"]] == [2, 1] assert report["limitations"] == [] @@ -362,7 +365,7 @@ def test_source_validation(): merge_tabular_sources([csv_source("a.csv", "A\n1\n"), csv_source("notes.docx", "A\n2\n")]) assert caught.value.code == "unsupported_format" with pytest.raises(TabularMergeError) as caught: - TabularMergeOptions(schema_policy="union") + TabularMergeOptions(schema_policy="crosswalk") assert caught.value.code == "invalid_options" diff --git a/functional_tests/test_tabular_merge_reconciliation.py b/functional_tests/test_tabular_merge_reconciliation.py new file mode 100644 index 000000000..29f5dc76f --- /dev/null +++ b/functional_tests/test_tabular_merge_reconciliation.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +# test_tabular_merge_reconciliation.py +""" +Functional test for reconciling differently structured spreadsheets in the merge engine. +Version: 0.261.219 +Implemented in: 0.261.219 + +This test ensures that functions_tabular_merge can merge files whose columns differ: +union keeps every column and leaves missing ones null, mapped keeps exactly the requested +columns and reports the rest, aliases line up renamed headers, a header row below a +title is honoured, every sheet of a workbook can be merged with its sheet name, files +that don't fit can be left out and reported, duplicate rows are removed exactly, rows +are sorted deterministically within a bound, and inspection describes each file and how +the files line up without failing on one unreadable file. +""" + +import io +import os +import sys + +import pytest + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "application", "single_app")) + +from test_support.versioning import assert_app_version_at_least + +from functions_tabular_merge import ( + TabularInspectOptions, + TabularMergeError, + TabularMergeLimits, + TabularMergeOptions, + TabularMergeSort, + TabularMergeSource, + inspect_tabular_sources, + merge_tabular_sources, + squash_header_name, +) + + +def csv_source(name, text, source_id=None): + data = text.encode("utf-8") + return TabularMergeSource(source_id or name, name, lambda data=data: data) + + +def bytes_source(name, data, source_id=None): + return TabularMergeSource(source_id or name, name, lambda data=data: data) + + +def xlsx_bytes(sheets): + from openpyxl import Workbook + + workbook = Workbook() + first = True + for title, rows, state in sheets: + worksheet = workbook.active if first else workbook.create_sheet() + first = False + worksheet.title = title + worksheet.sheet_state = state + for row in rows: + worksheet.append(row) + buffer = io.BytesIO() + workbook.save(buffer) + return buffer.getvalue() + + +def merged(sources, **options): + with merge_tabular_sources(sources, options=TabularMergeOptions(**options)) as result: + return result.columns, list(result.iter_records()), result.report, result.nullable_columns + + +def test_version_includes_reconciliation(): + assert_app_version_at_least("0.261.219") + + +def test_union_keeps_every_column_and_leaves_missing_values_null(): + first = csv_source("jan.csv", "Region,Amount\nEast,10\n") + second = csv_source("feb.csv", "Amount,Region,Channel\n5,West,Online\n") + third = csv_source("mar.csv", "Region\nNorth\n") + columns, rows, report, nullable = merged([first, second, third], schema_policy="union") + assert columns == ("Source File", "Region", "Amount", "Channel") + assert rows == [ + {"Source File": "jan.csv", "Region": "East", "Amount": "10", "Channel": None}, + {"Source File": "feb.csv", "Region": "West", "Amount": "5", "Channel": "Online"}, + {"Source File": "mar.csv", "Region": "North", "Amount": None, "Channel": None}, + ] + assert nullable == {"Amount", "Channel"} + assert report["nullable_columns"] == ["Amount", "Channel"] + assert report["sources"][1]["extra_columns"] == ["Channel"] + assert report["sources"][2]["missing_columns"] == ["Amount", "Channel"] + assert any("left blank" in text for text in report["limitations"]) + + +def test_union_renames_a_later_column_that_collides_with_the_file_name_column(): + first = csv_source("a.csv", "Region\nEast\n") + second = csv_source("b.csv", "Region,Source File\nWest,legacy\n") + columns, rows, report, _ = merged([first, second], schema_policy="union") + assert columns == ("Source File", "Region", "Source File (2)") + assert rows[1] == {"Source File": "b.csv", "Region": "West", "Source File (2)": "legacy"} + assert [warning["code"] for warning in report["warnings"]] == ["column_renamed"] + + +def test_aliases_line_up_renamed_headers_under_every_policy(): + first = csv_source("a.csv", "Customer ID,Amount\n001,10\n") + second = csv_source("b.csv", "cust_id,Total\n002,5\n") + aliases = {"Customer ID": ["cust_id", "CustomerID"], "Amount": ["Total"]} + columns, rows, report, nullable = merged([first, second], column_aliases=aliases) + assert columns == ("Source File", "Customer ID", "Amount") + assert [row["Customer ID"] for row in rows] == ["001", "002"] + assert report["sources"][1]["renamed_columns"] == [ + {"from": "cust_id", "to": "Customer ID"}, {"from": "Total", "to": "Amount"}, + ] + assert nullable == frozenset() + assert report["policy"]["column_aliases"] == {"Customer ID": ["cust_id", "CustomerID"], "Amount": ["Total"]} + + clash = csv_source("c.csv", "Customer ID,cust_id\n1,2\n") + with pytest.raises(TabularMergeError) as caught: + merged([first, clash], column_aliases=aliases) + assert caught.value.code == "duplicate_columns" + + +@pytest.mark.parametrize("aliases", [ + {"Customer ID": ["Amount"], "Amount": ["Total"]}, + {"Customer ID": ["Key"], "Account": ["key"]}, + {"Customer ID": []}, + {"Customer ID": "cust_id"}, + {"": ["x"]}, +]) +def test_invalid_aliases_are_refused(aliases): + with pytest.raises(TabularMergeError) as caught: + TabularMergeOptions(column_aliases=aliases) + assert caught.value.code == "invalid_options" + + +def test_identity_aliases_are_ignored(): + options = TabularMergeOptions(column_aliases={"Region": ["region", " REGION "]}) + assert options.column_aliases == () + + +def test_mapped_keeps_exactly_the_requested_columns_and_reports_the_rest(): + first = csv_source("a.csv", "cust_id,Amount,Notes\n001,10,hello\n") + second = csv_source("b.csv", "Amount,Region\n5,West\n") + columns, rows, report, nullable = merged( + [first, second], schema_policy="mapped", columns=["Customer ID", "Amount", "Region"], + column_aliases={"Customer ID": ["cust_id"]}, include_source_column=False, + ) + assert columns == ("Customer ID", "Amount", "Region") + assert rows == [ + {"Customer ID": "001", "Amount": "10", "Region": None}, + {"Customer ID": None, "Amount": "5", "Region": "West"}, + ] + assert nullable == {"Customer ID", "Region"} + assert report["sources"][0]["ignored_columns"] == ["Notes"] + assert report["sources"][0]["missing_columns"] == ["Region"] + assert any("left out" in text for text in report["limitations"]) + + +def test_mapped_fails_or_excludes_a_file_with_none_of_the_columns(): + first = csv_source("a.csv", "Amount\n10\n") + stray = csv_source("notes.csv", "Comment\nhello\n") + with pytest.raises(TabularMergeError) as caught: + merged([first, stray], schema_policy="mapped", columns=["Amount"]) + assert caught.value.code == "schema_mismatch" + assert caught.value.report["sources"][1]["reason"] == "no_matching_columns" + + columns, rows, report, _ = merged( + [first, stray], schema_policy="mapped", columns=["Amount"], on_incompatible="exclude", + ) + assert [row["Amount"] for row in rows] == ["10"] + assert report["sources"][1]["status"] == "excluded" + assert report["totals"]["excluded"] == 1 + + +def test_mapped_requires_columns_and_columns_require_mapped(): + with pytest.raises(TabularMergeError): + TabularMergeOptions(schema_policy="mapped") + with pytest.raises(TabularMergeError): + TabularMergeOptions(columns=["Amount"]) + with pytest.raises(TabularMergeError): + TabularMergeOptions(schema_policy="mapped", columns=["Amount", "amount"]) + with pytest.raises(TabularMergeError): + TabularMergeOptions(schema_policy="mapped", columns=["Amount"], column_aliases={"Region": ["Area"]}) + + +def test_excluding_files_whose_columns_differ_keeps_completeness_honest(): + first = csv_source("a.csv", "Region,Amount\nEast,10\n") + odd = csv_source("odd.csv", "Region,Total\nWest,5\n") + third = csv_source("c.csv", "Amount,Region\n7,North\n") + with pytest.raises(TabularMergeError) as caught: + merged([first, odd, third]) + assert caught.value.code == "schema_mismatch" + statuses = [entry["status"] for entry in caught.value.report["sources"]] + assert statuses == ["compatible", "schema_mismatch", "compatible"] + + columns, rows, report, _ = merged([first, odd, third], on_incompatible="exclude") + assert [row["Source File"] for row in rows] == ["a.csv", "c.csv"] + assert [entry["status"] for entry in report["sources"]] == ["merged", "excluded", "merged"] + assert report["sources"][1]["reason"] == "columns_differ" + assert report["sources"][1]["missing_columns"] == ["Amount"] + assert report["totals"]["excluded"] == 1 and report["totals"]["tables_merged"] == 2 + assert "1 of 3 file(s) or sheet(s) were left out" in report["limitations"][0] + + +def test_nothing_to_merge_when_every_file_is_left_out(): + empty = csv_source("empty.csv", "\n\n") + other = csv_source("blank.csv", " , \n") + with pytest.raises(TabularMergeError) as caught: + merged([empty, other], on_incompatible="exclude") + assert caught.value.code == "nothing_to_merge" + + +def test_header_row_skips_title_rows_in_csv_and_workbooks(): + titled = csv_source("titled.csv", "Quarterly sales\n\nRegion,Amount\nEast,10\n") + plain = bytes_source("plain.xlsx", xlsx_bytes([( + "Data", [["Report"], [None], ["Region", "Amount"], ["West", 5]], "visible", + )])) + columns, rows, report, _ = merged([titled, plain], header_row=3) + assert columns == ("Source File", "Region", "Amount") + assert [row["Region"] for row in rows] == ["East", "West"] + assert [entry["header_row"] for entry in report["sources"]] == [3, 3] + + blank_at_header = csv_source("gap.csv", "Title\n\n\nRegion\n") + with pytest.raises(TabularMergeError) as caught: + merged([titled, blank_at_header], header_row=3) + assert caught.value.code == "empty_source" + for invalid in (0, 1001, True, "3"): + with pytest.raises(TabularMergeError): + TabularMergeOptions(header_row=invalid) + + +def test_all_sheets_merges_every_visible_sheet_with_its_name(): + workbook = bytes_source("regions.xlsx", xlsx_bytes([ + ("East", [["Region", "Amount"], ["East", 1]], "visible"), + ("Hidden", [["Region", "Amount"], ["Secret", 99]], "hidden"), + ("Empty", [], "visible"), + ("West", [["Amount", "Region"], [2, "West"]], "visible"), + ])) + loose = csv_source("north.csv", "Region,Amount\nNorth,3\n") + columns, rows, report, nullable = merged([workbook, loose], sheets="all") + assert columns == ("Source File", "Source Sheet", "Region", "Amount") + assert rows == [ + {"Source File": "regions.xlsx", "Source Sheet": "East", "Region": "East", "Amount": "1"}, + {"Source File": "regions.xlsx", "Source Sheet": "West", "Region": "West", "Amount": "2"}, + {"Source File": "north.csv", "Source Sheet": None, "Region": "North", "Amount": "3"}, + ] + assert nullable == {"Source Sheet"} + assert report["sheet_column"] == "Source Sheet" + assert [(entry["sheet"], entry["status"], entry["reason"]) for entry in report["sources"]] == [ + ("East", "merged", None), ("Hidden", "skipped", "hidden_sheet"), ("Empty", "skipped", "no_header_row"), + ("West", "merged", None), (None, "merged", None), + ] + with pytest.raises(TabularMergeError): + TabularMergeOptions(sheet="East", sheets="all") + + +def test_all_sheets_with_exclusion_merges_only_the_matching_sheets(): + workbook = bytes_source("book.xlsx", xlsx_bytes([ + ("Data", [["Region", "Amount"], ["East", 1]], "visible"), + ("Notes", [["Comment"], ["Checked"]], "visible"), + ])) + other = bytes_source("other.xlsx", xlsx_bytes([("Data", [["Region", "Amount"], ["West", 2]], "visible")])) + _, rows, report, _ = merged([workbook, other], sheets="all", on_incompatible="exclude") + assert [(row["Source Sheet"], row["Region"]) for row in rows] == [("Data", "East"), ("Data", "West")] + assert [entry["status"] for entry in report["sources"]] == ["merged", "excluded", "merged"] + + +def test_a_missing_named_sheet_can_be_left_out(): + first = bytes_source("a.xlsx", xlsx_bytes([("Totals", [["A"], [1]], "visible")])) + second = bytes_source("b.xlsx", xlsx_bytes([("Other", [["A"], [2]], "visible")])) + with pytest.raises(TabularMergeError) as caught: + merged([first, second], sheet="Totals") + assert caught.value.code == "sheet_not_found" + _, rows, report, _ = merged([first, second], sheet="Totals", on_incompatible="exclude") + assert [row["A"] for row in rows] == ["1"] + assert report["sources"][1]["reason"] == "sheet_not_found" + + +def test_exact_duplicate_rows_are_removed_across_files_keeping_the_first(): + first = csv_source("a.csv", "Region,Amount\nEast,10\nWest,5\nEast,10\n") + second = csv_source("b.csv", "Amount,Region\n10,East\n7,North\n") + _, rows, report, _ = merged([first, second], dedupe="exact_rows") + assert [(row["Source File"], row["Region"]) for row in rows] == [ + ("a.csv", "East"), ("a.csv", "West"), ("b.csv", "North"), + ] + assert report["totals"]["duplicates_removed"] == 2 + assert [entry["duplicates_removed"] for entry in report["sources"]] == [1, 1] + assert [entry["rows"] for entry in report["sources"]] == [2, 1] + + +def test_keeping_the_last_duplicate_and_exact_counts_survive_rereads(): + first = csv_source("a.csv", "ID,Status\n1,open\n2,open\n") + second = csv_source("b.csv", "ID,Status\n1,closed\n3,open\n") + options = TabularMergeOptions(dedupe="key_columns", dedupe_columns=["id"], dedupe_keep="last") + with merge_tabular_sources([first, second], options=options) as result: + rows = list(result.iter_records()) + assert rows == [ + {"Source File": "a.csv", "ID": "2", "Status": "open"}, + {"Source File": "b.csv", "ID": "1", "Status": "closed"}, + {"Source File": "b.csv", "ID": "3", "Status": "open"}, + ] + assert result.row_count == 3 + assert list(result.iter_records()) == rows + report = result.report + assert [entry["rows"] for entry in report["sources"]] == [1, 2] + assert [entry["duplicates_removed"] for entry in report["sources"]] == [1, 0] + assert report["totals"]["rows"] == 3 and report["totals"]["duplicates_removed"] == 1 + + +def test_rows_with_blank_keys_are_never_treated_as_duplicates(): + first = csv_source("a.csv", "ID,Name\n,Ann\n,Bob\n7,Cy\n") + second = csv_source("b.csv", "ID,Name\n7,Dee\n") + _, rows, report, _ = merged([first, second], dedupe="key_columns", dedupe_columns=["ID"]) + assert [row["Name"] for row in rows] == ["Ann", "Bob", "Cy"] + assert report["totals"]["duplicates_removed"] == 1 + + +def test_union_rows_missing_a_later_column_match_rows_with_it_blank(): + first = csv_source("a.csv", "Region\nEast\n") + second = csv_source("b.csv", "Region,Channel\nEast,\nWest,Online\n") + _, rows, report, _ = merged([first, second], schema_policy="union", dedupe="exact_rows") + assert [(row["Region"], row["Channel"]) for row in rows] == [("East", None), ("West", "Online")] + assert report["totals"]["duplicates_removed"] == 1 + + +def test_dedupe_and_sort_columns_must_exist(): + first = csv_source("a.csv", "Region\nEast\n") + second = csv_source("b.csv", "Region\nWest\n") + with pytest.raises(TabularMergeError) as caught: + merged([first, second], dedupe="key_columns", dedupe_columns=["ID"]) + assert caught.value.code == "dedupe_column_not_found" + with pytest.raises(TabularMergeError) as caught: + merged([first, second], sort_by=[{"column": "Amount"}]) + assert caught.value.code == "sort_column_not_found" + with pytest.raises(TabularMergeError) as caught: + merged([first, second], schema_policy="union", sort_by=[{"column": "Amount"}]) + assert caught.value.code == "sort_column_not_found" + with pytest.raises(TabularMergeError): + TabularMergeOptions(dedupe_columns=["Region"]) + with pytest.raises(TabularMergeError): + TabularMergeOptions(dedupe="key_columns") + + +def test_sorting_by_number_puts_text_after_numbers_and_blanks_last(): + first = csv_source("a.csv", "Item,Amount\na,\"1,200\"\nb,n/a\nc,15\n") + second = csv_source("b.csv", "Item,Amount\nd,\ne,$3.5\nf,-2\n") + _, rows, _, _ = merged( + [first, second], include_source_column=False, + sort_by=[{"column": "amount", "value_type": "number"}], + ) + assert [row["Item"] for row in rows] == ["f", "e", "c", "a", "b", "d"] + _, rows, _, _ = merged( + [first, second], include_source_column=False, + sort_by=[TabularMergeSort("Amount", descending=True, value_type="number")], + ) + assert [row["Item"] for row in rows] == ["a", "c", "e", "f", "b", "d"] + + +def test_sorting_by_date_and_several_keys_is_stable(): + first = csv_source("a.csv", "Team,When,Seq\nB,2024-03-01,1\nA,2024-01-15T09:30:00,2\nA,2024-01-15,3\n") + second = csv_source("b.csv", "Team,When,Seq\nB,2023-12-31,4\nA,not a date,5\n") + _, rows, _, _ = merged( + [first, second], include_source_column=False, + sort_by=[{"column": "Team"}, {"column": "When", "value_type": "date"}], + ) + assert [row["Seq"] for row in rows] == ["3", "2", "5", "4", "1"] + + +def test_sort_with_keep_last_dedupe_and_the_sort_bound(): + first = csv_source("a.csv", "ID,Amount\n1,5\n2,9\n") + second = csv_source("b.csv", "ID,Amount\n1,7\n") + _, rows, _, _ = merged( + [first, second], include_source_column=False, dedupe="key_columns", dedupe_columns=["ID"], + dedupe_keep="last", sort_by=[{"column": "Amount", "value_type": "number"}], + ) + assert rows == [{"ID": "1", "Amount": "7"}, {"ID": "2", "Amount": "9"}] + + options = TabularMergeOptions(sort_by=[{"column": "ID"}]) + with pytest.raises(TabularMergeError) as caught: + merge_tabular_sources([first, second], options=options, limits=TabularMergeLimits(max_sort_rows=2)) + assert caught.value.code == "sort_limit_exceeded" + for invalid in ( + [{"column": "A"}, {"column": "B"}, {"column": "C"}, {"column": "D"}], + [{"column": "A"}, {"column": "a"}], + [{"column": "A", "value_type": "money"}], + [{"column": "A", "descending": "yes"}], + [{"column": "A", "order": "asc"}], + ): + with pytest.raises(TabularMergeError): + TabularMergeOptions(sort_by=invalid) + + +def test_inspection_describes_files_and_how_their_columns_line_up(): + first = csv_source("a.csv", "Customer ID,Amount,Region\n001," + "x" * 120 + ",East\n002,5,West\n\n") + second = csv_source("b.csv", "Region,Customer_ID,Amount\nNorth,003,7\n") + broken = bytes_source("broken.xlsx", b"PK\x03\x04not really a workbook") + report = inspect_tabular_sources([first, second, broken], options=TabularInspectOptions(sample_rows=1)) + assert report["version"] == "tabular-inspection-v1" + first_table = report["sources"][0]["tables"][0] + assert first_table["columns"] == ["Customer ID", "Amount", "Region"] + assert first_table["row_count"] == 2 + assert len(first_table["sample_rows"]) == 1 + assert len(first_table["sample_rows"][0][1]) == 80 and first_table["sample_rows"][0][1].endswith("\u2026") + assert report["sources"][2]["status"] == "problem" + assert report["sources"][2]["problem"]["code"] == "unreadable_workbook" + compatibility = report["compatibility"] + assert compatibility["tables"] == 2 + assert compatibility["identical_columns"] is False + assert compatibility["similar_columns"] == [{"names": ["Customer ID", "Customer_ID"]}] + assert compatibility["suggested_policy"] == "mapped" + assert {item["name"]: item["present_in"] for item in compatibility["columns"]} == { + "Customer ID": 1, "Amount": 2, "Region": 2, "Customer_ID": 1, + } + + +def test_inspection_suggests_by_name_for_identical_files_and_flags_title_rows(): + first = csv_source("a.csv", "Region,Amount\nEast,1\n") + second = csv_source("b.csv", "Amount,Region\n2,West\n") + report = inspect_tabular_sources([first, second]) + assert report["compatibility"]["identical_columns"] is True + assert report["compatibility"]["same_order"] is False + assert report["compatibility"]["suggested_policy"] == "by_name" + + titled = bytes_source("titled.xlsx", xlsx_bytes([ + ("Data", [["Sales report"], ["Region", "Amount"], ["East", 1]], "visible"), + ("Archive", [["Region", "Amount"]], "hidden"), + ])) + report = inspect_tabular_sources([titled], options=TabularInspectOptions(sheets="all")) + source = report["sources"][0] + assert source["sheets"] == [{"name": "Data", "visible": True}, {"name": "Archive", "visible": False}] + assert [table["status"] for table in source["tables"]] == ["inspected", "skipped"] + hint = source["tables"][0]["warnings"][0] + assert hint["code"] == "possible_title_row" and hint["suggested_header_row"] == 2 + unrelated = csv_source("c.csv", "Comment\nhi\n") + report = inspect_tabular_sources([first, unrelated]) + assert report["compatibility"]["suggested_policy"] == "union" + + +def test_inspection_drops_samples_to_stay_small(): + wide = ",".join(f"Column {index}" for index in range(200)) + values = ",".join("v" * 79 for _ in range(200)) + sources = [csv_source(f"{index}.csv", f"{wide}\n{values}\n{values}\n") for index in range(4)] + report = inspect_tabular_sources(sources, options=TabularInspectOptions(sample_rows=2)) + assert report["samples_omitted"] is True + assert all(not table["sample_rows"] for entry in report["sources"] for table in entry["tables"]) + assert report["limitations"][-1] == "Sample rows were left out to keep the inspection small." + + +def test_squashed_header_names_ignore_spacing_and_punctuation(): + assert squash_header_name(" Customer_ID ") == squash_header_name("customer id") == "customerid" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-q"])) diff --git a/functional_tests/test_v2_orchestration_merge_arguments.mjs b/functional_tests/test_v2_orchestration_merge_arguments.mjs new file mode 100644 index 000000000..a2371e136 --- /dev/null +++ b/functional_tests/test_v2_orchestration_merge_arguments.mjs @@ -0,0 +1,107 @@ +// test_v2_orchestration_merge_arguments.mjs +// Version: 0.261.219 +// Implemented in: 0.261.219 +// Executes how the V2 plan card states the settings of a spreadsheet merge or inspection step: +// every policy, alias, sheet, duplicate and sort setting is put in words, defaults are not +// restated, document selections are left to the chips, column and sheet names stay plain data, +// values the server would refuse are not put in words, and other capabilities are untouched. + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import './test_support/tsResolve.mjs'; + +globalThis.fetch = () => { + throw new Error('Merge argument helpers must not make network requests.'); +}; + +// The repository resolver must be registered before extensionless TypeScript imports load. +const { + TABULAR_INSPECT_CAPABILITY, + TABULAR_MERGE_CAPABILITY, + tabularArgumentEntries, +} = await import('../application/v2_ui/src/lib/orchestrationMerge.ts'); + +const HOSTILE = ' {{7*7}} [link](https://evil.example/)'; + +test('the capabilities match the server registry', () => { + assert.equal(TABULAR_MERGE_CAPABILITY, 'tabular_merge'); + assert.equal(TABULAR_INSPECT_CAPABILITY, 'tabular_inspect'); +}); + +test('a default merge states only how columns are matched', () => { + assert.deepEqual(tabularArgumentEntries('tabular_merge', { + document_ids: ['a', 'b'], doc_scope: 'all', schema_policy: 'by_name', + include_source_column: true, source_column_name: 'Source File', + }), [['columns', 'Same columns, in any order']]); +}); + +test('every reconciliation setting is put in words', () => { + assert.deepEqual(tabularArgumentEntries('tabular_merge', { + document_ids: ['a', 'b'], + doc_scope: 'group', + schema_policy: 'mapped', + columns: ['Customer ID', 'Amount'], + column_aliases: { 'Customer ID': ['cust_id', 'CustomerID'], Amount: ['Total'] }, + sheets: 'all', + header_row: 3, + include_source_column: true, + source_column_name: 'File', + on_incompatible: 'exclude', + dedupe: 'key_columns', + dedupe_columns: ['Customer ID'], + dedupe_keep: 'last', + sort_by: [{ column: 'Amount', value_type: 'number', descending: true }, { column: 'Customer ID' }], + }), [ + ['columns', 'Only the listed columns'], + ['keep', 'Customer ID, Amount'], + ['same column as', 'Customer ID \u2190 cust_id, CustomerID; Amount \u2190 Total'], + ['workspace', 'Group workspaces'], + ['sheets', 'Every visible sheet'], + ['header row', 'Row 3'], + ['file name column', 'File'], + ['files that don\u2019t fit', 'Left out and reported'], + ['duplicates', 'Remove rows with the same Customer ID, keeping the last'], + ['sort by', 'Amount (number, descending), Customer ID'], + ]); + assert.deepEqual(tabularArgumentEntries('tabular_merge', { + schema_policy: 'union', include_source_column: false, dedupe: 'exact_rows', sheet: 'Data', + }), [ + ['columns', 'Keep every column from every file'], + ['sheet', 'Data'], + ['file name column', 'Off'], + ['duplicates', 'Remove identical rows, keeping the first'], + ]); +}); + +test('names stay plain data', () => { + assert.deepEqual(tabularArgumentEntries('tabular_merge', { columns: [HOSTILE], sheet: HOSTILE }), [ + ['keep', HOSTILE], + ['sheet', HOSTILE], + ]); +}); + +test('values the server would refuse are not put in words', () => { + assert.deepEqual(tabularArgumentEntries('tabular_merge', { + schema_policy: 'toString', + doc_scope: '__proto__', + header_row: 0, + dedupe: 'key_columns', + dedupe_columns: [], + sort_by: [{ column: 'Amount', value_type: 'money' }], + column_aliases: ['not', 'an', 'object'], + }), []); + assert.deepEqual(tabularArgumentEntries('tabular_merge', null), []); + assert.deepEqual(tabularArgumentEntries('tabular_merge', { sort_by: [{ column: ' ' }] }), []); +}); + +test('inspection states its sheets, header row and sample size', () => { + assert.deepEqual(tabularArgumentEntries('tabular_inspect', { + document_ids: ['a'], doc_scope: 'all', sheets: 'all', header_row: 2, sample_rows: 5, + }), [['sheets', 'Every visible sheet'], ['header row', 'Row 2'], ['sample rows', '5']]); + assert.deepEqual(tabularArgumentEntries('tabular_inspect', { sample_rows: 50 }), []); +}); + +test('other capabilities keep the generic argument list', () => { + assert.equal(tabularArgumentEntries('document_analyze', { analysis_prompt: 'x' }), null); + assert.equal(tabularArgumentEntries('tabular_analyze', {}), null); +}); diff --git a/ui_tests/test_v2_orchestration_merge_arguments.py b/ui_tests/test_v2_orchestration_merge_arguments.py new file mode 100644 index 000000000..addbfcb04 --- /dev/null +++ b/ui_tests/test_v2_orchestration_merge_arguments.py @@ -0,0 +1,275 @@ +# test_v2_orchestration_merge_arguments.py +""" +Real-component browser tests for how the plan review states spreadsheet merge settings. +Version: 0.261.219 +Implemented in: 0.261.219 +Refs: microsoft/simplechat#1619 + +The production OrchestrationPlanCard and OrchestrationRunView run in Chromium with the +production CSS. Only HTTP boundaries are stubbed. The plan mirrors the server shape for a +tabular_inspect step, a compose step preparing a tabular_column_mapping_v1 mapping, and a +tabular_merge step whose union, alias, exclusion, duplicate and sort settings must read in +words, without raw argument names, with column names kept as plain text. + +Build CSS with the existing V2 build, keeping outputs in UI test artifacts: +npm --prefix .\\application\\v2_ui run build -- --outDir ..\\..\\ui_tests\\artifacts\\orchestration-plan-editor +Run: python -m pytest .\\ui_tests\\test_v2_orchestration_merge_arguments.py -q +""" + +import copy +import json +import re +import sys +from pathlib import Path +from urllib.parse import urlsplit + +import pytest +from playwright.sync_api import expect + +from test_v2_orchestration_plan_editor import ( # noqa: F401 + HARNESS, + ORIGIN, + connect_options, + editor_assets, + editor_browser, +) + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "functional_tests")) + +from test_support.versioning import assert_app_version_at_least # noqa: E402 + + +pytestmark = pytest.mark.ui +IMPLEMENTED_IN = "0.261.219" +CONVERSATION = "conversation-merge" +TURN = "merge-turn" +RUN = "/api/v2/orchestration/run" +RUNS = "/api/v2/orchestration/runs" +REQUEST = "Merge these sales files into one spreadsheet." +HOSTILE_COLUMN = ' cust id' +RAW_KEYS = [ + "schema_policy", "column_aliases", "on_incompatible", "dedupe", "dedupe_columns", "dedupe_keep", + "sort_by", "doc_scope", "include_source_column", "source_column_name", "header_row", + "sample_rows", "document_ids", +] + + +def plan_step(step_id, capability_id, role, arguments, *, inputs=None, outputs=None, title=None): + return { + "step_id": step_id, "capability_id": capability_id, "title": title or step_id.title(), + "rationale": "", "arguments": arguments, "depends_on": [], "inputs": inputs or {}, + "outputs": outputs or [], "optional": False, "enabled": True, "estimated_cost": "low", + "role": role, "status": "pending", + } + + +SERVER_PLAN = { + "kind": "plan", + "steps": [ + plan_step("inspect", "tabular_inspect", "gather", { + "document_ids": ["doc-east", "doc-west"], "doc_scope": "all", "sheets": "all", + "header_row": 2, "sample_rows": 5, + }, outputs=[{"name": "inspection", "kind": "structured-v1"}], title="Inspect spreadsheets"), + plan_step("merge", "tabular_merge", "reason", { + "document_ids": ["doc-east", "doc-west"], "doc_scope": "all", "schema_policy": "union", + "include_source_column": True, "source_column_name": "Source File", + "column_aliases": {"Customer ID": [HOSTILE_COLUMN, "CustomerID"]}, + "on_incompatible": "exclude", "dedupe": "key_columns", "dedupe_columns": ["Customer ID"], + "dedupe_keep": "last", + "sort_by": [{"column": "Amount", "value_type": "number", "descending": True}], + }, outputs=[{"name": "records", "kind": "records-v1"}, {"name": "report", "kind": "structured-v1"}], + title="Merge spreadsheets"), + plan_step("plain_merge", "tabular_merge", "reason", { + "document_ids": ["doc-east", "doc-west"], "doc_scope": "all", "schema_policy": "by_name", + "include_source_column": True, "source_column_name": "Source File", + }, outputs=[{"name": "records", "kind": "records-v1"}, {"name": "report", "kind": "structured-v1"}], + title="Merge matching spreadsheets"), + ], + "plan_id": "plan_merge_arguments", + "run_id": "run_merge_arguments", + "turn_id": TURN, + "revision": 0, + "conversation_id": CONVERSATION, + "user_id": "owner", + "planner_contract_version": 2, + "intent": {"summary": "", "complexity": "simple", "confidence": None}, + "assumptions": [], + "approval": { + "mode": "manual", "timeout_seconds": 10, "state": "pending", "approved_at": None, + "approved_by": None, "edited": False, + }, + "deliverables": [], + "validation": {"ok": True, "errors": [], "repairs": []}, + "inputs": { + "documents": [ + {"document_id": "doc-east", "title": "east.csv", "scope": "personal"}, + {"document_id": "doc-west", "title": "west.xlsx", "scope": "personal"}, + ], + "image_reference_documents": [], "image_reference_messages": [], "web": False, + "required_capabilities": [], "actions": [], "agent": None, + "model": {"model_deployment": "gpt-4o", "model_provider": "aoai"}, "prompt": None, + }, + "outputs": [ + {"kind": "structured-v1", "source_step_id": "inspect", "name": "inspection"}, + {"kind": "records-v1", "source_step_id": "merge", "name": "records"}, + ], + "status": "awaiting_approval", + "planner_model": "gpt-4o", + "planner": {"label": "gpt-4o", "source": "default"}, + "reasoning_adjustments": [], +} + + +def server_plan(): + return copy.deepcopy(SERVER_PLAN) + + +class MergeApi: + """The orchestration routes the card and run view call, recording unexpected requests.""" + + def __init__(self, assets): + self.assets = assets + self.unexpected = [] + + def handle(self, route): + request = route.request + parsed = urlsplit(request.url) + path = parsed.path + if f"{parsed.scheme}://{parsed.netloc}" != ORIGIN: + self.unexpected.append(request.url) + route.abort() + return + if request.method == "GET" and path in self.assets: + route.fulfill(path=str(self.assets[path])) + return + if path == RUN and request.method == "POST": + route.fulfill(content_type="text/event-stream", body=f"data: {json.dumps({'done': True})}\n\n") + return + if path == RUNS and request.method == "GET": + route.fulfill(json={"runs": []}) + return + if re.fullmatch(re.escape(RUNS) + r"/[^/]+/steps", path) and request.method == "GET": + route.fulfill(json={"steps": []}) + return + if path.startswith("/api/") and request.method == "GET": + route.fulfill(json={}) + return + if path == "/favicon.ico": + route.fulfill(status=204) + return + self.unexpected.append(f"{request.method} {path}") + route.fulfill(status=404, json={"error": "Unmocked request"}) + + +@pytest.fixture +def merge_ui(editor_browser, editor_assets): + context = editor_browser.new_context(viewport={"width": 1440, "height": 900}) + page = context.new_page() + api = MergeApi(editor_assets) + errors = [] + page.route("**/*", api.handle) + page.on("pageerror", lambda error: errors.append(str(error))) + page.on("console", lambda message: errors.append(message.text) if message.type == "error" else None) + try: + yield page, api + hostile = page.evaluate("() => window.__hostile ?? null") + assert hostile is None, hostile + finally: + context.close() + assert not api.unexpected, f"Unexpected browser requests: {api.unexpected}" + assert not errors, f"Unexpected browser errors: {errors}" + + +def mount_plan(page, api, plan, *, theme="light"): + page.goto(ORIGIN + HARNESS) + page.wait_for_function("() => Boolean(window.OrchHarness)") + for url in api.assets: + if url.endswith(".css"): + page.add_style_tag(url=ORIGIN + url) + page.evaluate( + """(spec) => { + const H = window.OrchHarness; + H.reset(); + document.documentElement.classList.toggle('dark', spec.theme === 'dark'); + H.stores.bootstrap.useBootstrapStore.setState({ data: { + version: '0.261.219', settings: {}, branding: { app_title: 'SimpleChat' }, + features: { enable_chat_orchestration: true }, + user: { id: 'owner', display_name: 'Merge Owner' }, + scope: { groups: [], public_workspaces: [] }, + orchestration: { enabled: true, allow_user_approval_override: true, capabilities: [] }, + catalogs: { models: [], agents: [], prompts: [] }, + } }); + H.stores.chat.useChatStore.setState({ + activeConversationId: spec.conversation, activeConversationKind: 'personal', + messagesLoading: false, messagesError: null, streaming: false, streamingContent: '', + streamError: null, thoughts: [], + conversations: [{ id: spec.conversation, title: 'Merge files' }], + messages: [{ id: 'user-1', conversation_id: spec.conversation, role: 'user', + content: spec.request, metadata: { orchestration_turn_id: spec.turn } }], + }); + const store = H.stores.orchestration.useOrchestrationStore.getState(); + store.setPlan(spec.conversation, spec.turn, spec.plan); + store.setActiveTurn(spec.conversation, spec.turn); + const props = { conversationId: spec.conversation, turnId: spec.turn }; + H.mount('mount-a', 'OrchestrationPlanCard', props, { strictMode: true }); + H.mount('mount-b', 'OrchestrationRunView', props, { strictMode: true }); + }""", + {"conversation": CONVERSATION, "turn": TURN, "plan": plan, "theme": theme, "request": REQUEST}, + ) + expect(page.locator("#mount-a").get_by_role("button", name="Approve and run the plan")).to_be_visible() + review = page.locator("#mount-b") + for step_id in ("inspect", "merge", "plain_merge"): + expect(review.locator(f'li[data-step-id="{step_id}"]')).to_be_visible() + return review + + +def terms(step): + arguments = step.get_by_test_id("orchestration-step-arguments") + return arguments.locator("dt").evaluate_all("(nodes) => nodes.map((node) => node.textContent?.trim() || '')") + + +def definitions(step): + arguments = step.get_by_test_id("orchestration-step-arguments") + return arguments.locator("dd").evaluate_all("(nodes) => nodes.map((node) => node.textContent?.trim() || '')") + + +def test_version_is_at_least_the_implementing_release(): + assert_app_version_at_least(IMPLEMENTED_IN) + + +@pytest.mark.parametrize("theme,width", [("light", 1440), ("dark", 390)]) +def test_merge_settings_read_in_words(merge_ui, theme, width): + page, api = merge_ui + page.set_viewport_size({"width": width, "height": 900}) + review = mount_plan(page, api, server_plan(), theme=theme) + + merge = review.locator('li[data-step-id="merge"]') + assert terms(merge) == ["columns", "same column as", "files that don\u2019t fit", "duplicates", "sort by"] + assert definitions(merge) == [ + "Keep every column from every file", + f"Customer ID \u2190 {HOSTILE_COLUMN}, CustomerID", + "Left out and reported", + "Remove rows with the same Customer ID, keeping the last", + "Amount (number, descending)", + ] + for tag in ("img", "script"): + expect(merge.locator(f"dd {tag}")).to_have_count(0) + box = merge.bounding_box() + assert box is not None and box["x"] >= 0 and box["x"] + box["width"] <= width, (box, width) + + plain = review.locator('li[data-step-id="plain_merge"]') + assert terms(plain) == ["columns"] + assert definitions(plain) == ["Same columns, in any order"] + + inspect = review.locator('li[data-step-id="inspect"]') + assert terms(inspect) == ["sheets", "header row", "sample rows"] + assert definitions(inspect) == ["Every visible sheet", "Row 2", "5"] + + for step_id in ("inspect", "merge", "plain_merge"): + shown = terms(review.locator(f'li[data-step-id="{step_id}"]')) + for key in RAW_KEYS: + assert key not in shown, (step_id, key, shown) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-q", "-p", "no:cacheprovider"])) From d80ebf6548f56df241cea408cc2cf9e10944e2d4 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 2 Oct 2026 22:40:58 -0400 Subject: [PATCH 03/13] Merge files as V2 workflow tasks (Phase 3, refs #1619) A workflow task can now merge many CSV and Excel files into one CSV or Excel file with code: Merge files is a new document action. Its files are chosen in order, or found at run time (every matching file, recent files, or the files File Sync changed) and merged in file-name order. Every file is authorized again, read through the access boundary, merged by the Phase 1-2 engine, rendered by the generated-file export framework and attached to the run once. No model runs. - functions_workflow_merge.py executes merges; the runner dispatches merge tasks, discovers run-time files, maps merge failures to WorkflowInputError, marks merge tasks replay-safe, and keys the artifact by run, task and bytes. - The merge contract raises reviewed MergeActionErrors that workflow saves show by task position; editor options say whether Merge is on and its limit. - Chat workflow proposals can carry merge tasks, with new repairable draft codes; the proposal card describes them as code that runs no model. - The V2 editor authors merge tasks; the classic editor routes them to V2. - Version 0.261.220, docs, release notes, and functional and UI tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../single_app/functions_document_actions.py | 184 +++- .../functions_orchestration_planner.py | 9 + ...ctions_orchestration_workflow_proposals.py | 2 + .../functions_orchestration_workflows.py | 8 +- .../functions_personal_workflows.py | 43 +- .../single_app/functions_tabular_merge.py | 11 +- .../single_app/functions_workflow_drafts.py | 173 ++- .../single_app/functions_workflow_editor.py | 24 +- .../single_app/functions_workflow_merge.py | 382 +++++++ .../single_app/functions_workflow_runner.py | 229 +++- .../js/workspace/workspace_workflows.js | 15 +- .../components/chat/WorkflowProposalCard.tsx | 15 +- .../workflows/WorkflowFileSyncFields.tsx | 2 +- .../workflows/WorkflowTaskFields.tsx | 542 +++++++++- application/v2_ui/src/lib/workflowEditor.ts | 540 +++++++++- application/v2_ui/src/lib/workflowFlow.ts | 2 +- .../v2_ui/src/lib/workflowProposals.ts | 33 + docs/admin/agents-actions.md | 10 +- docs/explanation/features/V2_FILE_MERGE.md | 33 +- .../V2_FILE_MERGE_PHASE_3_WORKFLOWS.md | 280 +++++ docs/explanation/release_notes.md | 21 + docs/guides/create-a-workflow.md | 62 +- docs/guides/merge-files.md | 36 +- ...chestration_workflow_propose_capability.py | 40 +- ...estration_workflow_results_off_golden.json | 9 + ...rchestration_workflow_runs_off_golden.json | 9 + .../test_v2_workflow_merge_task.mjs | 178 ++++ .../test_v2_workflow_proposal_merge.mjs | 110 ++ .../test_workflow_editor_options.py | 11 +- .../test_workflow_file_sync_prompt_context.py | 5 +- functional_tests/test_workflow_merge_task.py | 981 ++++++++++++++++++ .../test_workflow_task_document_actions.py | 16 +- ui_tests/fixtures/workflow_editor.py | 3 + ...v2_orchestration_workflow_proposal_card.py | 40 +- ui_tests/test_v2_workflow_merge_task.py | 105 ++ .../test_workflow_classic_advanced_guard.py | 46 +- 37 files changed, 4133 insertions(+), 78 deletions(-) create mode 100644 application/single_app/functions_workflow_merge.py create mode 100644 docs/explanation/features/V2_FILE_MERGE_PHASE_3_WORKFLOWS.md create mode 100644 functional_tests/test_v2_workflow_merge_task.mjs create mode 100644 functional_tests/test_v2_workflow_proposal_merge.mjs create mode 100644 functional_tests/test_workflow_merge_task.py create mode 100644 ui_tests/test_v2_workflow_merge_task.py diff --git a/application/single_app/config.py b/application/single_app/config.py index fa1818f47..8f3ca0aa3 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -100,7 +100,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.219" +VERSION = "0.261.220" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_document_actions.py b/application/single_app/functions_document_actions.py index 8b417f7eb..730a09a6e 100644 --- a/application/single_app/functions_document_actions.py +++ b/application/single_app/functions_document_actions.py @@ -78,13 +78,76 @@ }, } # Classic chat and its document-action payloads only know Analyze and Comparison. Merge is -# reached through V2 orchestration and, later, workflow tasks, so it is never offered to a -# caller that does not ask for it explicitly. +# reached through V2 orchestration and workflow tasks, so it is never offered to a caller +# that does not ask for it explicitly. DOCUMENT_ACTION_TYPES_WITH_PAYLOAD_SUPPORT = ( DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON, ) +# A workflow Merge task combines many files into one output file. The kind decides which +# files it reads and what it produces; later phases add kinds to MERGE_KINDS_AVAILABLE. +MERGE_KIND_TABULAR = 'tabular' +MERGE_KIND_WORKBOOK = 'workbook' +MERGE_KIND_PDF = 'pdf' +MERGE_KIND_DOCX = 'docx' +MERGE_KIND_PPTX = 'pptx' +MERGE_KINDS = (MERGE_KIND_TABULAR, MERGE_KIND_WORKBOOK, MERGE_KIND_PDF, MERGE_KIND_DOCX, MERGE_KIND_PPTX) +MERGE_KINDS_AVAILABLE = (MERGE_KIND_TABULAR,) +MERGE_KIND_LABELS = { + MERGE_KIND_TABULAR: 'spreadsheet', + MERGE_KIND_WORKBOOK: 'workbook', + MERGE_KIND_PDF: 'PDF', + MERGE_KIND_DOCX: 'Word', + MERGE_KIND_PPTX: 'PowerPoint', +} +MERGE_KIND_EXTENSIONS = { + MERGE_KIND_TABULAR: ('.csv', '.xlsx', '.xlsm', '.xls'), + MERGE_KIND_WORKBOOK: ('.csv', '.xlsx', '.xlsm', '.xls'), + MERGE_KIND_PDF: ('.pdf',), + MERGE_KIND_DOCX: ('.docx',), + MERGE_KIND_PPTX: ('.pptx',), +} +MERGE_KIND_OUTPUT_FORMATS = { + MERGE_KIND_TABULAR: ('csv', 'xlsx'), + MERGE_KIND_WORKBOOK: ('xlsx',), + MERGE_KIND_PDF: ('pdf',), + MERGE_KIND_DOCX: ('docx',), + MERGE_KIND_PPTX: ('pptx',), +} +MERGE_KIND_OPTION_KEYS = { + MERGE_KIND_TABULAR: ( + 'schema_policy', 'columns', 'column_aliases', 'sheet', 'sheets', 'header_row', + 'include_source_column', 'source_column_name', 'on_incompatible', 'dedupe', + 'dedupe_columns', 'dedupe_keep', 'sort_by', + ), + MERGE_KIND_WORKBOOK: ('sheets', 'sheet'), + MERGE_KIND_PDF: ('bookmarks',), + MERGE_KIND_DOCX: ('formatting', 'page_breaks', 'source_headings'), + MERGE_KIND_PPTX: ('formatting', 'sections'), +} +# Files are named by the user ("Changed", "All"...) or chosen from the workspace at run time. +MERGE_TARGET_MODE_CHANGED = 'changed' +MERGE_TARGET_MODES = ( + DOCUMENT_ACTION_TARGET_MODE_SELECTED, + DOCUMENT_ACTION_TARGET_MODE_ALL, + DOCUMENT_ACTION_TARGET_MODE_RECENT, + MERGE_TARGET_MODE_CHANGED, +) +MERGE_DEFAULT_RECENT_WINDOW_MINUTES = 60 +MERGE_OUTPUT_FILE_NAME_MAX_CHARS = 100 +_MERGE_FILE_NAME_FORBIDDEN = frozenset('/\\:*?"<>|') +_MERGE_DOCUMENT_FORMATTING = ('keep_source', 'use_first') +_MERGE_SHEET_MODES = ('first', 'all') + + +class MergeActionError(ValueError): + """A reviewed reason a Merge action is invalid; its text never repeats the caller's input. + + Workflow saves show it as written. Spreadsheet option errors from the merge engine stay plain + ``ValueError``s, because they can quote column names. + """ + def normalize_document_action_type(action_type): normalized_type = str(action_type or DOCUMENT_ACTION_TYPE_NONE).strip().lower() @@ -273,10 +336,120 @@ def _build_document_action_disabled_message(action_type): if action_type == DOCUMENT_ACTION_TYPE_COMPARISON: return 'Document comparison is currently disabled in admin settings.' if action_type == DOCUMENT_ACTION_TYPE_MERGE: - return 'Merging files is not available here.' + return 'File merging is turned off in admin settings, or is not available here.' return 'The selected document action is currently disabled in admin settings.' +def normalize_merge_output_file_name(value, output_format): + """A safe base name for a merged file without its extension, or '' when none is set.""" + if value is None or value == '': + return '' + if not isinstance(value, str): + raise MergeActionError('The merged file name must be text.') + name = ' '.join(value.split()) + suffix = f'.{output_format}' + if name.lower().endswith(suffix): + name = name[:-len(suffix)].rstrip() + if ( + len(name) > MERGE_OUTPUT_FILE_NAME_MAX_CHARS or name.startswith('.') + or any(character in _MERGE_FILE_NAME_FORBIDDEN or ord(character) < 32 for character in name) + ): + raise MergeActionError( + f'The merged file name must be at most {MERGE_OUTPUT_FILE_NAME_MAX_CHARS} characters, ' + 'must not start with a dot, and must not contain / \\ : * ? " < > |.' + ) + return name + + +def _normalize_merge_options(kind, options): + """Kind-specific merge options with unset values dropped; rules match chat merges.""" + if options is None: + return {} + if not isinstance(options, dict): + raise MergeActionError('Merge options must be an object.') + if set(options) - set(MERGE_KIND_OPTION_KEYS[kind]): + raise MergeActionError( + f"Some merge options don't apply to {MERGE_KIND_LABELS[kind]} merges. Remove them." + ) + cleaned = {key: copy.deepcopy(value) for key, value in options.items() if value is not None} + if kind == MERGE_KIND_TABULAR: + # The merge engine imports only the standard library and owns the spreadsheet rules. + from functions_tabular_merge import TabularMergeError, tabular_merge_options_from_arguments + + try: + tabular_merge_options_from_arguments(cleaned) + except TabularMergeError as exc: + raise ValueError(exc.message) from exc + return cleaned + for key in ('bookmarks', 'sections', 'page_breaks', 'source_headings'): + if key in cleaned and type(cleaned[key]) is not bool: + raise MergeActionError(f'The {key} merge option must be true or false.') + if 'formatting' in cleaned and cleaned['formatting'] not in _MERGE_DOCUMENT_FORMATTING: + raise MergeActionError('Formatting must be keep_source or use_first.') + if 'sheets' in cleaned and cleaned['sheets'] not in _MERGE_SHEET_MODES: + raise MergeActionError('Sheets must be first or all.') + if 'sheet' in cleaned: + if not isinstance(cleaned['sheet'], str) or not cleaned['sheet'].strip() or len(cleaned['sheet']) > 31: + raise MergeActionError('A sheet name must be 1 to 31 characters.') + if cleaned.get('sheets') == 'all': + raise MergeActionError('Name one sheet, or read all sheets, but not both.') + return cleaned + + +def _normalize_merge_action(source_action, normalized_action, max_documents=None): + """A workflow Merge task: which files, in which order, into which kind of file.""" + kind = str(source_action.get('merge_kind') or MERGE_KIND_TABULAR).strip().lower() + if kind not in MERGE_KINDS: + raise MergeActionError(f"Merge kind must be one of: {', '.join(MERGE_KINDS)}.") + if kind not in MERGE_KINDS_AVAILABLE: + raise MergeActionError(f'Merging {MERGE_KIND_LABELS[kind]} files is not available yet.') + target_mode = str(source_action.get('target_mode') or DOCUMENT_ACTION_TARGET_MODE_SELECTED).strip().lower() + if target_mode not in MERGE_TARGET_MODES: + raise MergeActionError(f"Merge files must be one of: {', '.join(MERGE_TARGET_MODES)}.") + doc_scope = str(source_action.get('doc_scope') or 'all').strip().lower() + if doc_scope not in {'all', 'personal', 'group', 'public'}: + raise MergeActionError('The merge workspace scope must be all, personal, group or public.') + resolved = bool(source_action.get('merge_targets_resolved')) + document_ids = normalize_search_id_list(source_action.get('document_ids')) + if target_mode == DOCUMENT_ACTION_TARGET_MODE_SELECTED: + if len(document_ids) < 2: + raise MergeActionError('Select at least two files to merge.') + if max_documents is not None and len(document_ids) > max_documents: + raise MergeActionError(f'A merge supports up to {max_documents} files at a time.') + elif not resolved: + # Files found at run time are never stored with the task. + document_ids = [] + # Files a run found are counted by the merge itself, which names the count and the limit. + output_formats = MERGE_KIND_OUTPUT_FORMATS[kind] + output_format = str(source_action.get('output_format') or output_formats[0]).strip().lower() + if output_format not in output_formats: + raise MergeActionError( + f"A {MERGE_KIND_LABELS[kind]} merge creates " + f"{' or '.join(value.upper() for value in output_formats)} files." + ) + normalized_action.update({ + 'merge_kind': kind, + 'target_mode': target_mode, + 'doc_scope': doc_scope, + 'active_group_ids': normalize_search_id_list(source_action.get('active_group_ids')), + 'active_public_workspace_id': normalize_search_id_list(source_action.get('active_public_workspace_id')), + 'document_ids': document_ids, + 'output_format': output_format, + 'merge_options': _normalize_merge_options(kind, source_action.get('merge_options')), + }) + if target_mode == DOCUMENT_ACTION_TARGET_MODE_RECENT: + window = source_action.get('recent_window_minutes') + normalized_action['recent_window_minutes'] = normalize_recent_document_window_minutes( + MERGE_DEFAULT_RECENT_WINDOW_MINUTES if window is None else window, + ) + file_name = normalize_merge_output_file_name(source_action.get('output_file_name'), output_format) + if file_name: + normalized_action['output_file_name'] = file_name + if resolved: + normalized_action['merge_targets_resolved'] = True + return normalized_action + + def _build_analyze_action(legacy_analyze=None): legacy_analyze = legacy_analyze if isinstance(legacy_analyze, dict) else {} if not legacy_analyze.get('enabled'): @@ -367,10 +540,11 @@ def normalize_document_action_config( } normalized_allowed_action_types.add(DOCUMENT_ACTION_TYPE_NONE) if action_type not in normalized_allowed_action_types: - raise ValueError(_build_document_action_disabled_message(action_type)) + message = _build_document_action_disabled_message(action_type) + raise MergeActionError(message) if action_type == DOCUMENT_ACTION_TYPE_MERGE else ValueError(message) if action_type == DOCUMENT_ACTION_TYPE_MERGE: - raise ValueError(_build_document_action_disabled_message(action_type)) + return _normalize_merge_action(source_action, normalized_action, resolved_max_documents) if action_type == DOCUMENT_ACTION_TYPE_ANALYZE: if 'analysis_options' in source_action or 'transformation_spec' in source_action: diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index 8fb1c42b0..58eba095f 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -285,6 +285,15 @@ def resolve_planner_client(settings): words such as "this week" can stay relative. - optionally "alerts" {"mode":"every_run"|"failures_only","severity":"info"|"low"}, "run_as" "self"|"none", and "durable": true. +- A task can merge CSV or Excel files with code instead of a model by adding "merge": + {"files":"inputs"|"changed"|"all"|"recent","output_format":"csv"|"xlsx","file_name","options"} + and no agent runner. + inputs merges the task's two or more input documents in order; changed merges the files a + file_sync trigger added or changed; all merges every CSV and Excel file in the user's personal + workspace; recent merges those added or changed in the last "recent_window_minutes". "options" + takes tabular_merge's column, sheet, duplicate and sort settings, with "column_aliases" written + as [{"column","aliases":[...]}]. Propose one when the user wants files merged on a schedule or on + every sync, or more files than one chat merge allows; it needs no task_actions. Refer to agents, documents and File Sync sources only by their workflow_planning.catalog handles, and never write a record id, model or endpoint into a blueprint. "task_actions" lists, for each task in order, the action kinds it needs from the capability's input schema, or [] when it needs none. diff --git a/application/single_app/functions_orchestration_workflow_proposals.py b/application/single_app/functions_orchestration_workflow_proposals.py index 587d2fd96..8a679cb57 100644 --- a/application/single_app/functions_orchestration_workflow_proposals.py +++ b/application/single_app/functions_orchestration_workflow_proposals.py @@ -158,10 +158,12 @@ _CONFLICT_DRAFT_CODES = frozenset({ 'quota_exceeded', 'workflow_conflict', 'agent_unavailable', 'reference_unknown', 'reference_unauthorized', 'file_sync_source_unavailable', 'workflow_unavailable', 'workflow_definition_conflict', 'workflow_deleted', + 'merge_unavailable', }) _INVALID_DRAFT_CODES = frozenset({ 'blueprint_invalid', 'unsupported_field', 'too_many_tasks', 'trigger_invalid', 'cadence_below_minimum', 'invalid_workflow', 'invalid_workflow_definition', 'invalid_workflow_settings', 'invalid_workflow_alerts', + 'merge_inputs_required', 'merge_trigger_required', 'merge_options_invalid', 'merge_runner_invalid', }) _INVALID_WORKFLOW_MESSAGE = 'Invalid workflow settings. Review the task, runner, trigger, and document inputs.' _SAFE_POINTER = re.compile(r'^(/[A-Za-z0-9_-]{1,64}){0,8}$') diff --git a/application/single_app/functions_orchestration_workflows.py b/application/single_app/functions_orchestration_workflows.py index 727cb49cc..48cc6ed7f 100644 --- a/application/single_app/functions_orchestration_workflows.py +++ b/application/single_app/functions_orchestration_workflows.py @@ -488,6 +488,10 @@ def _proposal_summary(blueprint, task_actions, planning, used, request_time_zone document_names.get(handle) or 'Document' for handle in task.get('inputs') or () if isinstance(handle, str) ], + **({'merge': { + 'files': task['merge'].get('files'), + 'output_format': task['merge'].get('output_format') or 'csv', + }} if isinstance(task.get('merge'), dict) else {}), }) m365_sources, can_send, required = set(), False, False @@ -529,8 +533,10 @@ def _uncovered_task(blueprint, task_actions, planning): def _uses_default_model(blueprint): + # A merge task merges files with code, so it never needs the default model. return any( - isinstance(task, dict) and _runner_agent(task) is None for task in blueprint.get('tasks') or () + isinstance(task, dict) and _runner_agent(task) is None and not task.get('merge') + for task in blueprint.get('tasks') or () ) diff --git a/application/single_app/functions_personal_workflows.py b/application/single_app/functions_personal_workflows.py index 589ca9b2a..32ba43c00 100644 --- a/application/single_app/functions_personal_workflows.py +++ b/application/single_app/functions_personal_workflows.py @@ -21,7 +21,10 @@ from functions_debug import debug_print from functions_document_actions import ( DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_CONTEXT_WORKFLOW, + MERGE_TARGET_MODE_CHANGED, + MergeActionError, build_analyze_config, get_document_action_max_documents_by_type, get_enabled_document_action_types, @@ -320,6 +323,9 @@ def _normalize_workflow_tasks( raw_document_action = default_document_action try: normalized_task['document_action'] = task_document_action_normalizer(raw_document_action) + except WorkflowPublicValidationError as exc: + # Reviewed messages stay public; the task is named by position, never by its name. + raise WorkflowPublicValidationError(f'Workflow task {index + 1}: {exc.public_message}') from exc except ValueError as exc: raise ValueError(f'Workflow task {index + 1} ({name}): {exc}') from exc @@ -351,12 +357,34 @@ def _normalize_workflow_error_handling(workflow_data, existing_workflow=None): } +def _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets): + """Merging the files a sync changed needs a File Sync trigger that passes them on.""" + if ( + isinstance(action_payload, dict) + and str(action_payload.get('type') or '').strip().lower() == DOCUMENT_ACTION_TYPE_MERGE + and str(action_payload.get('target_mode') or '').strip().lower() == MERGE_TARGET_MODE_CHANGED + and not allow_empty_file_sync_targets + ): + raise WorkflowPublicValidationError( + 'Merging the files a sync changed needs a File Sync trigger that uses changed documents.' + ) + + +def _normalize_merge_errors_for_save(normalize): + """Run a document action normalizer, showing a Merge action's reviewed reason as written.""" + try: + return normalize() + except MergeActionError as exc: + raise WorkflowPublicValidationError(str(exc)) from exc + + def _normalize_document_action_config(workflow_data, existing_workflow=None, allow_empty_file_sync_targets=False, settings=None): workflow_data = workflow_data if isinstance(workflow_data, dict) else {} existing_workflow = existing_workflow if isinstance(existing_workflow, dict) else {} settings = get_settings() if settings is None else settings action_payload = workflow_data.get('document_action') + _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets) if allow_empty_file_sync_targets and isinstance(action_payload, dict): action_type = str(action_payload.get('type') or '').strip().lower() document_ids = action_payload.get('document_ids') if isinstance(action_payload.get('document_ids'), list) else [] @@ -372,12 +400,12 @@ def _normalize_document_action_config(workflow_data, existing_workflow=None, all DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), ) normalized_action['document_ids'] = [] return normalized_action - return normalize_document_action_config( + return _normalize_merge_errors_for_save(lambda: normalize_document_action_config( action_payload=action_payload, existing_action=existing_workflow.get('document_action'), legacy_analyze=workflow_data.get('analyze') or existing_workflow.get('analyze'), @@ -385,8 +413,8 @@ def _normalize_document_action_config(workflow_data, existing_workflow=None, all DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), - ) + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), + )) def _normalize_task_document_action_config(action_payload, allow_empty_file_sync_targets=False, settings=None, @@ -394,11 +422,12 @@ def _normalize_task_document_action_config(action_payload, allow_empty_file_sync """Normalize a single workflow task's document action payload.""" source_settings = settings if isinstance(settings, dict) else get_settings() action_payload = action_payload if isinstance(action_payload, dict) else {'type': 'none'} + _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets) max_documents_by_type = get_document_action_max_documents_by_type( DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=source_settings, ) - allowed_action_types = get_enabled_document_action_types(settings=source_settings) + allowed_action_types = get_enabled_document_action_types(settings=source_settings, include_merge=True) if action_payload.get('target_mode') == 'current_item': if ( not allow_current_item or action_payload.get('type') != DOCUMENT_ACTION_TYPE_ANALYZE @@ -429,11 +458,11 @@ def _normalize_task_document_action_config(action_payload, allow_empty_file_sync normalized_action['document_ids'] = [] return normalized_action - return normalize_document_action_config( + return _normalize_merge_errors_for_save(lambda: normalize_document_action_config( action_payload=action_payload, max_documents_by_type=max_documents_by_type, allowed_action_types=allowed_action_types, - ) + )) def _normalize_file_sync_config(user_id, workflow_data, existing_workflow=None, sanitize_source=None): diff --git a/application/single_app/functions_tabular_merge.py b/application/single_app/functions_tabular_merge.py index c011977c1..46129066d 100644 --- a/application/single_app/functions_tabular_merge.py +++ b/application/single_app/functions_tabular_merge.py @@ -1,9 +1,10 @@ # functions_tabular_merge.py """Deterministic merging and inspection of CSV and Excel files. -Version: 0.261.219 +Version: 0.261.220 Implemented in: 0.261.218 Reconciliation policies, sheet modes, duplicate removal, sorting and inspection added in: 0.261.219 +Single-file merges for workflow files found at run time (min_sources=1) added in: 0.261.220 The engine is pure: it receives already-authorized byte loaders, never resolves documents, settings, storage, routes, or models, and performs no model work. @@ -787,18 +788,22 @@ def merge_tabular_sources( limits: Optional[TabularMergeLimits] = None, cancel_requested: Optional[Callable[[], bool]] = None, on_progress: Optional[Callable[[dict], None]] = None, + min_sources: int = 2, ) -> TabularMergeResult: """Append the rows of the sources, in the given order, into one table. The first file (or sheet) defines the column order for ``by_name`` and ``exact_order``; ``union`` adds each new column the first time a file has it; ``mapped`` uses exactly the requested columns. Under ``on_incompatible="fail"`` a mismatch reads every remaining - header so the report lists all mismatches, then nothing is merged. + header so the report lists all mismatches, then nothing is merged. ``min_sources`` is 1 + only for a scheduled merge of whatever files arrived, which may be a single file. """ + if min_sources not in (1, 2): + raise ValueError("min_sources must be 1 or 2.") options = options or TabularMergeOptions() limits = limits or TabularMergeLimits() sources = list(sources or ()) - _validate_sources(sources, limits) + _validate_sources(sources, limits, minimum=min_sources) merger = _Merger(options, limits, len(sources), cancel_requested) try: for index, source in enumerate(sources): diff --git a/application/single_app/functions_workflow_drafts.py b/application/single_app/functions_workflow_drafts.py index 3501b260e..ec80fb31d 100644 --- a/application/single_app/functions_workflow_drafts.py +++ b/application/single_app/functions_workflow_drafts.py @@ -36,7 +36,12 @@ from content_screening.contracts import DocumentHeldError from functions_appinsights import log_event -from functions_document_actions import DOCUMENT_ACTION_TYPE_ANALYZE, get_enabled_document_action_types +from functions_document_actions import ( + DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, + get_enabled_document_action_types, + is_document_action_enabled, +) from functions_file_sync import ( FILE_SYNC_SCOPE_GROUP, FILE_SYNC_SCOPE_PERSONAL, @@ -110,6 +115,9 @@ BLUEPRINT_RUN_AS = ('self', 'none') BLUEPRINT_SCOPE_TYPES = (FILE_SYNC_SCOPE_PERSONAL, FILE_SYNC_SCOPE_GROUP, FILE_SYNC_SCOPE_PUBLIC) BLUEPRINT_HANDLE_KINDS = ('documents', 'agents', 'sources') +# Where a merge task's files come from: its inputs, a sync, or the user's personal workspace. +BLUEPRINT_MERGE_FILES = ('inputs', 'changed', 'all', 'recent') +BLUEPRINT_MERGE_FILE_NAME_MAX_LENGTH = 100 DRAFT_MAX_ERRORS = 10 DRAFT_MESSAGE_MAX_LENGTH = 240 @@ -130,6 +138,14 @@ 'file_sync_source_unavailable': 'This File Sync source is not available to you.', 'workflow_conflict': 'A different workflow already uses this id.', 'workflows_unavailable': 'Personal workflows are not available for this account.', + 'merge_unavailable': 'File merging is turned off by an administrator. Remove merge from this task.', + 'merge_inputs_required': 'A merge of input documents needs at least two input documents, in merge order.', + 'merge_trigger_required': 'Merging the files a sync changed needs a File Sync trigger.', + 'merge_runner_invalid': 'A merge task merges files with code, so it has no agent runner. Remove the runner.', + 'merge_options_invalid': ( + 'These merge options cannot be used together: mapped needs columns, key_columns needs ' + 'dedupe_columns, a sheet name cannot be combined with all sheets, and a column is named once.' + ), } DRAFT_UNAVAILABLE_REFERENCE_MESSAGE = 'A document or source in this blueprint is not available to you.' DRAFT_HANDLE_MAP_ERROR = 'The workflow draft handle map is malformed.' @@ -441,6 +457,70 @@ def _trigger_schema(): } +def _merge_schema(): + return { + 'type': 'object', + 'additionalProperties': False, + 'required': ['files'], + 'description': ( + 'Makes this task merge CSV or Excel files into one file with code instead of running a ' + "model. files: inputs merges the task's inputs (two or more document handles) in order; " + 'changed merges the files a File Sync trigger added or changed; all merges every CSV and ' + "Excel file in the user's personal workspace; recent merges those added in the last " + 'recent_window_minutes. Rows are appended; rows are never matched on a key.' + ), + 'properties': { + 'files': {'enum': list(BLUEPRINT_MERGE_FILES)}, + 'output_format': {'enum': ['csv', 'xlsx']}, + 'file_name': _text_schema(BLUEPRINT_MERGE_FILE_NAME_MAX_LENGTH), + 'recent_window_minutes': {'type': 'integer', 'minimum': 1, 'maximum': 1440}, + 'options': {'$ref': '#/$defs/merge_options'}, + }, + } + + +def _merge_options_schema(): + name = {'type': 'string', 'minLength': 1, 'maxLength': 256} + names = {'type': 'array', 'items': name, 'minItems': 1, 'uniqueItems': True} + return { + 'type': 'object', + 'additionalProperties': False, + 'description': 'The same column, sheet, duplicate and sort settings a chat merge accepts.', + 'properties': { + 'schema_policy': {'enum': ['by_name', 'exact_order', 'union', 'mapped']}, + 'columns': {**names, 'maxItems': 255}, + # A list of closed objects rather than a map, so every object in the blueprint stays closed. + 'column_aliases': { + 'type': 'array', 'minItems': 1, 'maxItems': 256, + 'items': { + 'type': 'object', 'additionalProperties': False, 'required': ['column', 'aliases'], + 'properties': {'column': name, 'aliases': {**names, 'maxItems': 32}}, + }, + }, + 'sheet': {'type': 'string', 'minLength': 1, 'maxLength': 31}, + 'sheets': {'enum': ['first', 'all']}, + 'header_row': {'type': 'integer', 'minimum': 1, 'maximum': 1000}, + 'include_source_column': {'type': 'boolean'}, + 'source_column_name': {'type': 'string', 'minLength': 1, 'maxLength': 128}, + 'on_incompatible': {'enum': ['fail', 'exclude']}, + 'dedupe': {'enum': ['none', 'exact_rows', 'key_columns']}, + 'dedupe_columns': {**names, 'maxItems': 16}, + 'dedupe_keep': {'enum': ['first', 'last']}, + 'sort_by': { + 'type': 'array', 'maxItems': 3, + 'items': { + 'type': 'object', 'additionalProperties': False, 'required': ['column'], + 'properties': { + 'column': name, + 'descending': {'type': 'boolean'}, + 'value_type': {'enum': ['text', 'number', 'date']}, + }, + }, + }, + }, + } + + def _build_blueprint_schema(): return { '$schema': 'https://json-schema.org/draft/2020-12/schema', @@ -471,6 +551,7 @@ def _build_blueprint_schema(): 'type': 'array', 'maxItems': BLUEPRINT_TASK_MAX_INPUTS, 'uniqueItems': True, 'items': {'$ref': '#/$defs/handle'}, }, + 'merge': {'$ref': '#/$defs/merge'}, }, }, }, @@ -502,6 +583,8 @@ def _build_blueprint_schema(): ], }, 'file_sync_schedule': _file_sync_schedule_schema(), + 'merge': _merge_schema(), + 'merge_options': _merge_options_schema(), }, } @@ -798,6 +881,85 @@ def _alert_fields(alerts, workflow_id): } +def _merge_options(options, *, aliases_as_pairs=False): + """A blueprint's merge options in the stored form: column aliases become a column-to-names map. + + Validation reads the aliases as ordered pairs, so a column named twice is reported rather + than silently overwritten. + """ + options = copy.deepcopy(options or {}) + if 'column_aliases' in options: + pairs = [(entry['column'], list(entry['aliases'])) for entry in options['column_aliases']] + options['column_aliases'] = pairs if aliases_as_pairs else dict(pairs) + return options + + +def _merge_document_action(merge, task, handles): + """The workflow Merge action a blueprint task describes; its files are never reference documents.""" + action = { + 'type': DOCUMENT_ACTION_TYPE_MERGE, + 'merge_kind': 'tabular', + 'output_format': merge.get('output_format', 'csv'), + 'document_ids': [], + 'active_group_ids': [], + 'active_public_workspace_id': [], + } + if merge.get('file_name'): + action['output_file_name'] = merge['file_name'] + if merge.get('options'): + action['merge_options'] = _merge_options(merge['options']) + files = merge['files'] + if files == 'inputs': + documents = [handles['documents'][handle] for handle in task.get('inputs') or []] + scopes = {document['scope_type'] for document in documents} + action.update({ + 'target_mode': 'selected', + 'document_ids': [document['document_id'] for document in documents], + 'doc_scope': next(iter(scopes)) if len(scopes) == 1 else 'all', + 'active_group_ids': list(dict.fromkeys( + document['scope_id'] for document in documents if document['scope_type'] == FILE_SYNC_SCOPE_GROUP + )), + 'active_public_workspace_id': list(dict.fromkeys( + document['scope_id'] for document in documents if document['scope_type'] == FILE_SYNC_SCOPE_PUBLIC + )), + }) + elif files == 'changed': + action.update({'target_mode': 'changed', 'doc_scope': 'all'}) + else: + # A chat-created workflow is personal, so run-time discovery stays in the personal workspace. + action.update({'target_mode': files, 'doc_scope': FILE_SYNC_SCOPE_PERSONAL}) + if files == 'recent' and merge.get('recent_window_minutes'): + action['recent_window_minutes'] = merge['recent_window_minutes'] + return action + + +def _merge_errors(blueprint, settings): + """Merge task rules that read no storage, so a planner can repair them in one round.""" + errors = [] + merging = [(index, task) for index, task in enumerate(blueprint['tasks']) if task.get('merge')] + if not merging: + return errors + if not is_document_action_enabled(DOCUMENT_ACTION_TYPE_MERGE, settings=settings): + return [draft_error('merge_unavailable', ('tasks', index, 'merge')) for index, _ in merging] + # The merge engine imports only the standard library; blueprints share its option rules. + from functions_tabular_merge import TabularMergeError, tabular_merge_options_from_arguments + + for index, task in merging: + merge = task['merge'] + if (task.get('runner') or {}).get('type') == 'agent': + errors.append(draft_error('merge_runner_invalid', ('tasks', index, 'runner'))) + if merge['files'] == 'inputs' and len(task.get('inputs') or []) < 2: + errors.append(draft_error('merge_inputs_required', ('tasks', index, 'inputs'))) + if merge['files'] == 'changed' and blueprint['trigger']['type'] != 'file_sync': + errors.append(draft_error('merge_trigger_required', ('tasks', index, 'merge', 'files'))) + try: + tabular_merge_options_from_arguments(_merge_options(merge.get('options'), aliases_as_pairs=True)) + except TabularMergeError: + # Engine messages can quote column names from the blueprint; draft errors never echo input. + errors.append(draft_error('merge_options_invalid', ('tasks', index, 'merge', 'options'))) + return errors + + def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, enabled): trigger = blueprint['trigger'] analyze_changed_documents = ( @@ -809,7 +971,8 @@ def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, en tasks = [] for index, task in enumerate(blueprint['tasks']): task_reference_ids = [] - for handle in task.get('inputs') or []: + merge = task.get('merge') + for handle in [] if merge else task.get('inputs') or []: if handle not in reference_ids: reference_ids[handle] = _derived_id(workflow_id, 'reference', handle) references.append({'id': reference_ids[handle], 'name': handle, **handles['documents'][handle]}) @@ -820,7 +983,9 @@ def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, en else: # The workflow runs on the default model; a blueprint never names one. task_runner = {'type': 'inherit'} - if index == 0 and analyze_changed_documents: + if merge: + document_action = _merge_document_action(merge, task, handles) + elif index == 0 and analyze_changed_documents: # The first task of a File Sync workflow analyzes the documents each sync changed. document_action = {'type': DOCUMENT_ACTION_TYPE_ANALYZE, 'document_ids': []} else: @@ -1028,6 +1193,7 @@ def _checked_blueprint(user_id, blueprint, handles, *, workflow_id, settings, us # Collected together, so one repair round can fix every problem at once. errors = [ *_schedule_errors(prepared['trigger'], settings), + *_merge_errors(prepared, settings), *_agent_errors(uses, handles, user_id=user_id, settings=settings, reader=reader), *_document_errors(uses, handles, user_id=user_id, resolver=resolver), *_source_errors(uses, handles, user_id=user_id, settings=settings, user_info=user_info), @@ -1059,6 +1225,7 @@ def check_workflow_blueprint(blueprint, *, settings, handle_names=None): if len(prepared['tasks']) > task_limit: return None, [draft_error('too_many_tasks', ('tasks',), f'This workflow can have up to {task_limit} tasks.')] errors = list(_schedule_errors(prepared['trigger'], settings)) + errors.extend(_merge_errors(prepared, settings)) if handle_names is not None: names = {kind: set(handle_names.get(kind) or ()) for kind in BLUEPRINT_HANDLE_KINDS} errors = [*_unknown_handle_errors(prepared, names), *errors] diff --git a/application/single_app/functions_workflow_editor.py b/application/single_app/functions_workflow_editor.py index e4e3adea2..1ee456be7 100644 --- a/application/single_app/functions_workflow_editor.py +++ b/application/single_app/functions_workflow_editor.py @@ -26,7 +26,8 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks agents, endpoints, default_model=None, max_loop_items=WORKFLOW_LOOP_ITEMS_DEFAULT, max_repeat_iterations=WORKFLOW_REPEAT_ITERATIONS_DEFAULT, - min_schedule_interval_seconds=WORKFLOW_MIN_SCHEDULE_INTERVAL_DEFAULT): + min_schedule_interval_seconds=WORKFLOW_MIN_SCHEDULE_INTERVAL_DEFAULT, + merge=None): if scope_type not in {"personal", "group"}: raise ValueError("Unsupported workflow editor scope.") agent_options = [ @@ -65,7 +66,7 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks }) default_model = default_model or {} default_model_valid = bool(default_model.get("valid")) - return { + options = { "definition_version": WORKFLOW_DEFINITION_VERSION, "supported_definition_versions": [1, 2, 3], "supported_node_kinds": ["task", "if", "route", "for_each", "collect", "repeat_until"], @@ -100,11 +101,24 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks "loop_eligible": default_model_valid, }, } + if merge is not None: + # Whether a task may merge files, and how many one run may merge, as the save will check. + options["document_actions"] = {"merge": { + "enabled": bool(merge.get("enabled")), + "workflow_max_documents": int(merge["workflow_max_documents"]), + }} + return options def get_workflow_editor_options(user_id, settings, *, group_id=""): # Existing stores own agent/model eligibility and initialize app services. # Import them only at this already-authorized request boundary. + from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_WORKFLOW, + DOCUMENT_ACTION_TYPE_MERGE, + get_document_action_max_documents, + is_document_action_enabled, + ) from functions_group import assert_group_role from functions_group_workflows import ( GROUP_WORKFLOW_MEMBER_ROLES, @@ -141,4 +155,10 @@ def get_workflow_editor_options(user_id, settings, *, group_id=""): max_loop_items=get_workflow_max_loop_items(settings), max_repeat_iterations=get_workflow_max_repeat_iterations(settings), min_schedule_interval_seconds=get_workflow_min_schedule_interval_seconds(settings), + merge={ + "enabled": is_document_action_enabled(DOCUMENT_ACTION_TYPE_MERGE, settings=settings), + "workflow_max_documents": get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, + ), + }, ) diff --git a/application/single_app/functions_workflow_merge.py b/application/single_app/functions_workflow_merge.py new file mode 100644 index 000000000..3c5b565ca --- /dev/null +++ b/application/single_app/functions_workflow_merge.py @@ -0,0 +1,382 @@ +# functions_workflow_merge.py +"""Workflow Merge tasks: combine many authorized files into one downloadable file. + +Version: 0.261.220 +Implemented in: 0.261.220 + +A workflow task whose document action is ``merge`` runs here instead of a model or agent. +The runner says which files to merge: the files selected on the task, in that order, or +files found when the run starts (every matching file in a workspace, recently added files, +or the files a File Sync run changed), ordered by file name. This module authorizes every +file again through the source manifest, reads each original through the screening-aware +byte reader, merges them with code, renders one CSV or Excel file through the shared +generated-file export framework, and attaches it to the run's conversation. No model is +called. A resumed run repeats the merge from its first file; merging is deterministic, so +the resulting file is the same, and the artifact idempotency key keeps one copy. +""" + +import os + +from functions_appinsights import log_event +from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_WORKFLOW, + DOCUMENT_ACTION_TARGET_MODE_ALL, + DOCUMENT_ACTION_TARGET_MODE_RECENT, + DOCUMENT_ACTION_TARGET_MODE_SELECTED, + MERGE_KIND_EXTENSIONS, + MERGE_KIND_LABELS, + MERGE_KIND_TABULAR, + MERGE_TARGET_MODE_CHANGED, +) +from functions_generated_export_contracts import ( + GeneratedFileExportError, + GeneratedFileExportReadiness, + GeneratedFileExportRequest, +) +from functions_mixed_source_orchestration import AUTHORIZATION_STATUS_AUTHORIZED, SOURCE_KIND_TABULAR +from functions_orchestration_merge import build_tabular_merge_sources, tabular_merge_limits +from functions_tabular_merge import ( + TabularMergeCancelled, + TabularMergeError, + merge_tabular_sources, + tabular_merge_options_from_arguments, +) + + +WORKFLOW_MERGE_CAPABILITY = "file_merge" +WORKFLOW_MERGE_DEFAULT_FILE_NAME = "merged" +_EXPORT_PROFILES = { + "csv": "exact_tabular_records_v1", + "xlsx": "exact_tabular_workbook_v1", +} +_MAX_LISTED_FILES = 20 +_MAX_LISTED_COLUMNS = 12 +_EXPORT_FAILURES = { + "record_limit": "The merged table has more rows than one file can hold. Merge fewer files per run.", + "value_limit": "The merged table is larger than one generated file can hold. Merge fewer files per run.", +} +_OFFICE_FAILURES = { + "limit_exceeded": ( + "The merged table is too large for an Excel file (at most 1,048,575 rows, 5,000,000 cells " + "and 32 MB). Choose CSV output instead." + ), +} + + +class WorkflowMergeError(ValueError): + """A merge task that cannot run; the message is shown on the failed task.""" + + +class WorkflowMergeAccessError(PermissionError): + """A file the merge needs is not available to the workflow's owner; the message is fixed text.""" + + +class WorkflowMergeCancelled(RuntimeError): + """The run was cancelled while the merge was in progress.""" + + +class _MergedRecordsSource: + """The merged rows as a complete record source for the shared CSV and XLSX renderers.""" + + kind = "records" + + def __init__(self, merge_result): + self._result = merge_result + self.record_count = merge_result.row_count + self.columns = tuple(merge_result.columns) + self.readiness = GeneratedFileExportReadiness() + + def iter_records(self): + return self._result.iter_records() + + def recheck(self): + if self._result.row_count != self.record_count or tuple(self._result.columns) != self.columns: + raise GeneratedFileExportError("invalid_source", "The merged rows changed while rendering.") + + +def merge_output_file_name(action_config, output_format): + """The file name a merge task creates, from its configured base name.""" + base = str((action_config or {}).get("output_file_name") or "").strip() or WORKFLOW_MERGE_DEFAULT_FILE_NAME + return f"{base}.{output_format}" + + +def _target_description(target_mode): + return { + DOCUMENT_ACTION_TARGET_MODE_ALL: "No files in the workspace matched this merge", + DOCUMENT_ACTION_TARGET_MODE_RECENT: "No files were added or updated in the merge's time window", + MERGE_TARGET_MODE_CHANGED: "No new or changed files arrived with this sync", + }.get(target_mode, "No files matched this merge") + + +def _nothing_to_merge_result(target_mode): + return { + "reply": f"{_target_description(target_mode)}, so there was nothing to merge and no file was created.", + "generated_analysis_artifacts": [], + "generated_tabular_outputs": [], + "token_usage": {}, + "model_deployment_name": None, + "provider": None, + "merge_summary": {"status": "nothing_to_merge", "files": 0, "rows": 0}, + } + + +def _listed(values, limit): + values = [str(value) for value in values] + shown = ", ".join(values[:limit]) + if len(values) > limit: + shown += f" and {len(values) - limit} more" + return shown + + +def build_merge_reply(report, file_name, *, skipped_files=(), unavailable_files=()): + """A short Markdown summary of what the merge produced and what it left out.""" + totals = report.get("totals") or {} + lines = [ + f"Merged **{totals.get('rows', 0):,} row(s)** from **{totals.get('sources', 0)} file(s)** into " + f"**{file_name}** with {totals.get('columns', 0)} column(s).", + "", + ] + details = [] + if totals.get("duplicates_removed"): + details.append(f"Removed {totals['duplicates_removed']:,} duplicate row(s).") + left_out = [ + entry for entry in report.get("sources") or [] + if entry.get("status") in ("excluded", "skipped") + ] + if left_out: + labels = [ + f"{entry.get('file_name')}" + (f" ({entry.get('sheet')})" if entry.get("sheet") else "") + for entry in left_out + ] + details.append(f"Left out {len(left_out)} file(s) or sheet(s): {_listed(labels, _MAX_LISTED_FILES)}.") + if skipped_files: + details.append( + f"Skipped {len(skipped_files)} file(s) that aren't the right type: " + f"{_listed(skipped_files, _MAX_LISTED_FILES)}." + ) + if unavailable_files: + details.append( + f"Skipped {len(unavailable_files)} file(s) that aren't available right now: " + f"{_listed(unavailable_files, _MAX_LISTED_FILES)}." + ) + nullable = [name for name in report.get("nullable_columns") or [] if name != report.get("sheet_column")] + if nullable: + details.append( + f"Columns some files don't have are left blank for their rows: {_listed(nullable, _MAX_LISTED_COLUMNS)}." + ) + if details: + lines.extend(f"- {detail}" for detail in details) + lines.append("") + merged = [ + entry.get("file_name") for entry in report.get("sources") or [] if entry.get("status") == "merged" + ] + lines.append(f"Files merged, in order: {_listed(dict.fromkeys(merged), _MAX_LISTED_FILES)}.") + return "\n".join(lines).strip() + + +def execute_workflow_merge( + action_config, + settings, + *, + user_id, + conversation_id, + run_id, + task_id="", + collect_documents, + resolve_manifest, + render_file, + publish_file, + byte_reader=None, + cancel_requested=None, + report_progress=None, +): + """Merge a workflow task's files and attach the merged file to the run conversation. + + ``collect_documents(action_config, extensions, max_documents)`` returns document ids, in + merge order, for files found at run time. ``resolve_manifest`` authorizes ids again. + ``render_file(source, request, max_output_bytes)`` and ``publish_file(...)`` are the + shared export and artifact services, injected so the merge stays testable. + """ + action_config = action_config if isinstance(action_config, dict) else {} + kind = action_config.get("merge_kind") or MERGE_KIND_TABULAR + if kind != MERGE_KIND_TABULAR: + raise WorkflowMergeError(f"Merging {MERGE_KIND_LABELS.get(kind, kind)} files is not available yet.") + target_mode = action_config.get("target_mode") or DOCUMENT_ACTION_TARGET_MODE_SELECTED + output_format = action_config.get("output_format") or "csv" + if output_format not in _EXPORT_PROFILES: + raise WorkflowMergeError("Spreadsheet merges create CSV or Excel files.") + limits = tabular_merge_limits(settings, execution_context=DOCUMENT_ACTION_CONTEXT_WORKFLOW) + extensions = MERGE_KIND_EXTENSIONS[kind] + selected = target_mode == DOCUMENT_ACTION_TARGET_MODE_SELECTED + + def check_cancel(): + if callable(cancel_requested) and cancel_requested(): + raise WorkflowMergeCancelled("The merge was cancelled.") + + check_cancel() + if target_mode in (DOCUMENT_ACTION_TARGET_MODE_ALL, DOCUMENT_ACTION_TARGET_MODE_RECENT): + document_ids = list(collect_documents(action_config, extensions, limits.max_sources)) + else: + document_ids = list(action_config.get("document_ids") or []) + if not document_ids: + if selected: + raise WorkflowMergeError("Select at least two files to merge.") + return _nothing_to_merge_result(target_mode) + if len(document_ids) > limits.max_sources: + raise WorkflowMergeError( + f"{len(document_ids)} files match this merge; at most {limits.max_sources} can be merged in one " + "workflow run. Narrow the files or ask an administrator to raise the Merge workflow file limit." + ) + + check_cancel() + manifest = resolve_manifest( + document_ids, user_id=user_id, conversation_id=conversation_id, + doc_scope=action_config.get("doc_scope", "all"), + active_group_ids=action_config.get("active_group_ids"), + active_public_workspace_ids=action_config.get("active_public_workspace_id"), + cancel_requested=cancel_requested, request_correlation_id=run_id, + ) + if ( + not isinstance(manifest, list) + or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids + ): + raise WorkflowMergeAccessError("The files in this merge could not be authorized.") + unavailable = [ + str(source.get("file_name") or source.get("document_id")) for source in manifest + if source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED + ] + if unavailable and selected: + raise WorkflowMergeAccessError( + "A file selected for this merge is no longer available to the workflow's owner." + ) + + usable = [] + skipped = [] + for source in manifest: + if source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED: + continue + extension = os.path.splitext(str(source.get("file_name") or ""))[1].strip().lower() + if extension in extensions and source.get("source_kind") == SOURCE_KIND_TABULAR: + usable.append(source) + else: + skipped.append(str(source.get("file_name") or source.get("document_id"))) + if skipped and selected: + raise WorkflowMergeError( + f"{_listed(skipped, 5)} isn't a CSV or Excel file (.csv, .xlsx, .xlsm or .xls), so it can't be merged." + ) + if not usable: + return _nothing_to_merge_result(target_mode) + + def progress(update): + if callable(report_progress): + report_progress(update) + + sources = build_tabular_merge_sources(usable, user_id, byte_reader=byte_reader) + file_name = merge_output_file_name(action_config, output_format) + try: + options = tabular_merge_options_from_arguments(action_config.get("merge_options") or {}) + with merge_tabular_sources( + sources, options=options, limits=limits, min_sources=2 if selected else 1, + cancel_requested=lambda: bool(callable(cancel_requested) and cancel_requested()), + on_progress=progress, + ) as merged: + check_cancel() + report = merged.report + max_output_bytes = _max_artifact_bytes(settings) + request = GeneratedFileExportRequest(output_format, _EXPORT_PROFILES[output_format]) + with render_file(_MergedRecordsSource(merged), request, max_output_bytes) as rendered: + check_cancel() + # A replay of the same merge reuses its file; one over files that changed since gets its own. + content_key = str(getattr(rendered, "content_sha256", "") or "") + artifact = publish_file( + user_id=user_id, conversation_id=conversation_id, file_name=file_name, + output_format=output_format, rendered=rendered, row_count=merged.row_count, + summary=( + f"Merged {merged.row_count:,} row(s) from {report['totals']['sources']} file(s)." + ), + idempotency_key=f"workflow-merge:{run_id}:{task_id or 'task'}:{content_key}", + ) + except TabularMergeCancelled as exc: + raise WorkflowMergeCancelled("The merge was cancelled.") from exc + except TabularMergeError as exc: + # The owner's own file and column names make the failure actionable on the task. + raise WorkflowMergeError(exc.message) from exc + except GeneratedFileExportError as exc: + raise WorkflowMergeError( + _EXPORT_FAILURES.get(exc.code, "The merged file couldn't be created.") + ) from exc + except ValueError as exc: + # Office renderer failures carry a stable code; the module loads only when rendering. + if type(exc).__name__ != "OfficeRenderError": + raise + raise WorkflowMergeError( + _OFFICE_FAILURES.get(getattr(exc, "code", None), "The merged Excel file couldn't be created.") + ) from exc + + log_event( + "[WORKFLOW_MERGE] Merged workflow files.", + extra={ + "user_id": user_id, "run_id": run_id, "task_id": task_id, "files": len(usable), + "rows": report["totals"]["rows"], "output_format": output_format, + "excluded": report["totals"].get("excluded", 0), "skipped": len(skipped), + }, + ) + return { + "reply": build_merge_reply( + report, artifact.get("file_name") or file_name, skipped_files=skipped, unavailable_files=unavailable, + ), + "generated_analysis_artifacts": [artifact], + "generated_tabular_outputs": [artifact] if output_format == "csv" else [], + "token_usage": {}, + "model_deployment_name": None, + "provider": None, + "merge_summary": { + "status": "merged", + "files": report["totals"]["sources"], + "rows": report["totals"]["rows"], + "columns": report["totals"]["columns"], + "duplicates_removed": report["totals"].get("duplicates_removed", 0), + "excluded": report["totals"].get("excluded", 0), + "skipped": len(skipped) + len(unavailable), + "file_name": artifact.get("file_name") or file_name, + }, + } + + +def _max_artifact_bytes(settings): + try: + megabytes = max(1, int((settings or {}).get("max_generated_chat_artifact_size_mb", 500))) + except (TypeError, ValueError): + megabytes = 500 + return megabytes * 1024 * 1024 + + +def collect_merge_candidates(documents, extensions, max_documents): + """Current, matching documents ordered by file name; more than the limit fails, never truncates.""" + matches = {} + for document in documents or (): + document_id = str(document.get("id") or document.get("document_id") or "").strip() + file_name = str(document.get("file_name") or "").strip() + if not document_id or document_id in matches: + continue + if os.path.splitext(file_name)[1].lower() not in extensions: + continue + matches[document_id] = (file_name.casefold(), file_name, document_id) + if len(matches) > max_documents: + raise WorkflowMergeError( + f"More than {max_documents} files match this merge; at most {max_documents} can be merged in one " + "workflow run. Narrow the files or ask an administrator to raise the Merge workflow file limit." + ) + return [document_id for _, _, document_id in sorted(matches.values())] + + +__all__ = [ + "WORKFLOW_MERGE_CAPABILITY", + "WorkflowMergeAccessError", + "WorkflowMergeCancelled", + "WorkflowMergeError", + "build_merge_reply", + "collect_merge_candidates", + "execute_workflow_merge", + "merge_output_file_name", +] diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index d8da23154..bc9d9472b 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -118,9 +118,11 @@ DOCUMENT_ACTION_TARGET_MODE_RECENT, DOCUMENT_ACTION_TYPE_COMPARISON, DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_TYPE_NONE, DOCUMENT_ACTION_TYPE_SEARCH, DEFAULT_RECENT_DOCUMENT_WINDOW_MINUTES, + MERGE_TARGET_MODE_CHANGED, build_analyze_config, get_document_action_config, get_document_action_max_documents, @@ -6440,7 +6442,8 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru raw_agent_citations = list(result.get('agent_citations') or []) xsd_generation_contract = result.get('_xsd_generation_contract') generated_file_output = None - if not xsd_generation_contract: + # A merge task's merged file is its output; its instructions never ask for a second file. + if not xsd_generation_contract and not result.get('merge_summary'): generated_file_output = _maybe_create_workflow_generated_file_output( workflow=workflow, conversation_id=conversation.get('id'), @@ -6790,7 +6793,7 @@ def _get_document_action_config(workflow): DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), ) @@ -7691,8 +7694,21 @@ def _format_workflow_file_sync_context(file_sync_result): def _apply_file_sync_changed_documents_to_action(action_config, changed_document_ids, group_ids, public_workspace_ids): - """Point an analyze document action at the documents File Sync just changed.""" + """Point an analyze document action, or a merge of changed files, at the documents File Sync just changed.""" action_config = action_config if isinstance(action_config, dict) else {} + if action_config.get('type') == DOCUMENT_ACTION_TYPE_MERGE: + if action_config.get('target_mode') != MERGE_TARGET_MODE_CHANGED: + return None + # A merge with no changed files reports that there was nothing to merge. + updated_merge = dict(action_config) + updated_merge.update({ + 'document_ids': list(changed_document_ids), + 'doc_scope': 'all', + 'active_group_ids': list(group_ids), + 'active_public_workspace_id': list(public_workspace_ids), + 'merge_targets_resolved': True, + }) + return updated_merge if action_config.get('type') != DOCUMENT_ACTION_TYPE_ANALYZE: return None @@ -9809,6 +9825,194 @@ def record_native_token_usage(token_usage): } +def _query_merge_candidate_documents(container, scope_clause, scope_parameters, extensions, cutoff_ts=None): + parameters = list(scope_parameters) + extension_clauses = [] + for index, extension in enumerate(extensions): + parameter = f'@merge_extension_{index}' + extension_clauses.append(f'ENDSWITH(LOWER(c.file_name), {parameter})') + parameters.append({'name': parameter, 'value': extension}) + query = f"SELECT * FROM c WHERE {scope_clause} AND ({' OR '.join(extension_clauses)})" + if cutoff_ts is not None: + query += ' AND c._ts >= @cutoff_ts' + parameters.append({'name': '@cutoff_ts', 'value': cutoff_ts}) + return list(container.query_items(query=query, parameters=parameters, enable_cross_partition_query=True)) + + +def _collect_merge_workflow_documents(workflow, action_config, settings, extensions): + """Current documents of a merge's file types in the task's authorized workspace scopes. + + Every match is returned; the merge orders them and refuses more than its file limit + rather than silently merging only some of them. + """ + user_id = str(workflow.get('user_id') or '').strip() + workflow_group_id = _get_workflow_group_id(workflow) + doc_scope = normalize_search_scope(action_config.get('doc_scope')) + active_group_ids = normalize_search_id_list(action_config.get('active_group_ids')) + if workflow_group_id: + assert_group_role( + user_id, + workflow_group_id, + allowed_roles=("Owner", "Admin", "DocumentManager", "User"), + ) + active_group_ids = [workflow_group_id] + doc_scope = 'group' + else: + active_group_ids = _resolve_recent_authorized_group_ids(user_id, active_group_ids) + active_public_workspace_ids = normalize_search_id_list(action_config.get('active_public_workspace_id')) + if not workflow_group_id: + active_public_workspace_ids = _resolve_recent_authorized_public_workspace_ids( + user_id, active_public_workspace_ids, + ) + cutoff_ts = None + if action_config.get('target_mode') == DOCUMENT_ACTION_TARGET_MODE_RECENT: + window_minutes = _coerce_workflow_recent_window_minutes(action_config.get('recent_window_minutes')) + cutoff_ts = int(datetime.now(timezone.utc).timestamp()) - (window_minutes * 60) + + documents = [] + if doc_scope in {'personal', 'all'} and not workflow_group_id and user_id: + documents.extend(_query_merge_candidate_documents( + cosmos_user_documents_container, 'c.user_id = @user_id', + [{'name': '@user_id', 'value': user_id}], extensions, cutoff_ts, + )) + if doc_scope in {'group', 'all'}: + for group_id in active_group_ids: + documents.extend(_query_merge_candidate_documents( + cosmos_group_documents_container, 'c.group_id = @group_id', + [{'name': '@group_id', 'value': group_id}], extensions, cutoff_ts, + )) + if doc_scope in {'public', 'all'} and not workflow_group_id: + for workspace_id in active_public_workspace_ids: + documents.extend(_query_merge_candidate_documents( + cosmos_public_documents_container, 'c.public_workspace_id = @workspace_id', + [{'name': '@workspace_id', 'value': workspace_id}], extensions, cutoff_ts, + )) + return select_current_documents(documents) + + +def _throttled_cancellation_check(check, interval_seconds=2.0): + """Ask whether a run was cancelled at most once per interval; a cancellation sticks.""" + state = {'checked_at': None, 'cancelled': False} + + def cancelled(): + now = time.monotonic() + if not state['cancelled'] and ( + state['checked_at'] is None or now - state['checked_at'] >= interval_seconds + ): + state['checked_at'] = now + state['cancelled'] = bool(check()) + return state['cancelled'] + + return cancelled + + +def _execute_document_merge_workflow( + workflow, + action_config, + settings, + conversation_id='', + run_id=None, + thought_tracker=None, +): + """Run a task's Merge document action: many files into one CSV or Excel file, without a model.""" + # The merge engine and its upload path load only when a workflow merges files. + from functions_simplechat_operations import upload_generated_file_artifact_stream_for_user + from functions_workflow_merge import ( + WORKFLOW_MERGE_CAPABILITY, + WorkflowMergeAccessError, + WorkflowMergeCancelled, + WorkflowMergeError, + collect_merge_candidates, + execute_workflow_merge, + ) + + user_id = str(workflow.get('user_id') or '').strip() + task = workflow.get('active_task') if isinstance(workflow.get('active_task'), dict) else {} + activity_key = f"file-merge:{run_id}:{task.get('id') or 'task'}" + + def collect(action, extensions, max_documents): + documents = _collect_merge_workflow_documents(workflow, action, settings, extensions) + return collect_merge_candidates(documents, extensions, max_documents) + + def render(source, request, max_output_bytes): + return build_generated_file_export(source=source, export_request=request, max_output_bytes=max_output_bytes) + + def publish(*, user_id, conversation_id, file_name, output_format, rendered, row_count, summary, idempotency_key): + upload_result = upload_generated_file_artifact_stream_for_user( + user_id, + conversation_id, + file_name, + rendered.file_content, + rendered.size_bytes, + capability=WORKFLOW_MERGE_CAPABILITY, + output_format=output_format, + summary=summary, + artifact_idempotency_key=idempotency_key, + ) + metadata = build_generated_file_artifact_metadata( + { + 'file_name': file_name, 'output_format': output_format, 'capability': WORKFLOW_MERGE_CAPABILITY, + 'summary': summary, 'row_count': row_count, + }, + upload_result, + conversation_id, + ) + if not metadata: + raise RuntimeError('The merged file could not be attached to the workflow run.') + return metadata + + def progress(update): + _add_workflow_activity_thought( + thought_tracker, + workflow, + run_id, + step_type='file_merge', + content=f"Merged file {update.get('index')} of {update.get('total')}", + detail=f"rows={update.get('rows', 0)}", + activity_key=activity_key, + kind='file_merge', + title='Merge files', + status='running', + ) + + cancel_requested = _throttled_cancellation_check( + lambda: _is_workflow_run_cancellation_requested(workflow, run_id), + ) + try: + result = execute_workflow_merge( + action_config, + settings, + user_id=user_id, + conversation_id=conversation_id, + run_id=run_id, + task_id=str(task.get('id') or ''), + collect_documents=collect, + resolve_manifest=resolve_analysis_source_manifest, + render_file=render, + publish_file=publish, + cancel_requested=cancel_requested, + report_progress=progress, + ) + except WorkflowMergeCancelled as exc: + raise WorkflowRunCancelledError(WORKFLOW_RUN_CANCELLED_MESSAGE) from exc + except (WorkflowMergeError, WorkflowMergeAccessError) as exc: + # The task shows why, in terms of the owner's own files; the same merge would fail again. + raise WorkflowInputError(str(exc)) from exc + _add_workflow_activity_thought( + thought_tracker, + workflow, + run_id, + step_type='file_merge', + content='Merged files', + detail=f"files={(result.get('merge_summary') or {}).get('files', 0)}", + activity_key=activity_key, + kind='file_merge', + title='Merge files', + status='completed', + ) + return result + + def _execute_document_action_workflow( workflow, settings, @@ -10457,7 +10661,20 @@ def _execute_workflow_dispatch( execution_workflow = workflow_search_context.get('workflow') or execution_workflow document_action = _get_document_action_config(execution_workflow) - if document_action.get('type') in {DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON}: + if document_action.get('type') == DOCUMENT_ACTION_TYPE_MERGE: + execution_result = _execute_cancelable_workflow_step( + execution_workflow, + run_id, + lambda: _execute_document_merge_workflow( + execution_workflow, + document_action, + settings, + conversation_id=conversation_id, + run_id=run_id, + thought_tracker=thought_tracker, + ), + ) + elif document_action.get('type') in {DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON}: _raise_if_workflow_run_cancelled(execution_workflow, run_id) document_action = _resolve_recent_document_action_targets(execution_workflow, document_action, settings) execution_workflow = _apply_runtime_document_action_config(execution_workflow, document_action) @@ -10963,11 +11180,13 @@ def raise_if_cancelled(): 'runner': runner_audit, **({'iteration_inputs': durable.iteration_inputs} if flow_runner and durable.iteration_path else {}), } + # A merge only reads files and attaches its file under an idempotency key, so a + # durable run may repeat it instead of pausing for a review of external actions. replay_safe = ( attempt_workflow.get('runner_type') == 'model' and not attempt_workflow.get('chat_capabilities_enabled') and (attempt_workflow.get('document_action') or {}).get('type') in {None, 'none'} - ) + ) or (attempt_workflow.get('document_action') or {}).get('type') == 'merge' def dispatch_task(): nonlocal analysis_checkpoints if structured_definition: diff --git a/application/single_app/static/js/workspace/workspace_workflows.js b/application/single_app/static/js/workspace/workspace_workflows.js index abb0afabf..8e5636482 100644 --- a/application/single_app/static/js/workspace/workspace_workflows.js +++ b/application/single_app/static/js/workspace/workspace_workflows.js @@ -224,6 +224,8 @@ const DOCUMENT_ACTION_NONE = "none"; const DOCUMENT_ACTION_SEARCH = "search"; const DOCUMENT_ACTION_ANALYZE = "analyze"; const DOCUMENT_ACTION_COMPARISON = "comparison"; +// Merge files is edited only in V2; the classic page lists it and refuses to edit it. +const DOCUMENT_ACTION_MERGE = "merge"; const DOCUMENT_ANALYSIS_MODE_COMBINED = "combined"; const DOCUMENT_ANALYSIS_MODE_PER_DOCUMENT = "per_document"; const DOCUMENT_ANALYSIS_TARGET_SELECTED = "selected"; @@ -3166,6 +3168,10 @@ function getWorkflowDocumentActionSummary(workflow) { return `Compare one source to ${rightCount || 0} ${rightCount === 1 ? "target" : "targets"}`; } + if (config.type === DOCUMENT_ACTION_MERGE) { + return "Merge files"; + } + return "No document action"; } @@ -4084,7 +4090,8 @@ function resetWorkflowForm() { } // Calendar schedules come first: the classic form edits only fixed intervals, and saving one here -// would replace the calendar schedule. +// would replace the calendar schedule. The classic form has no Merge files action either, so it +// would save a merge task as a task with no document action. function workflowNativeEditorReason(workflow) { if (!workflow || typeof workflow !== "object") { return ""; @@ -4095,6 +4102,12 @@ function workflowNativeEditorReason(workflow) { if ((workflow.definition_version !== undefined && workflow.definition_version !== 1) || workflow.flow !== undefined) { return "advanced data flow"; } + const tasks = Array.isArray(workflow.tasks) ? workflow.tasks : []; + if ([workflow.document_action, ...tasks.map((task) => task?.document_action)].some( + (action) => normalizeText(action?.type).toLowerCase() === "merge" + )) { + return "a Merge files task"; + } return ""; } diff --git a/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx b/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx index f0e1c5cfe..b8c266902 100644 --- a/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx +++ b/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx @@ -33,6 +33,7 @@ import { type WorkflowProposalMode, type WorkflowProposalState, type WorkflowProposalSummary, + workflowProposalMergeText as mergeTaskText, } from '../../lib/workflowProposals'; const PERSONAL_SCOPE: WorkflowScope = { type: 'personal' }; @@ -246,18 +247,22 @@ function ProposalDetails({ proposal }: { proposal: WorkflowProposal }) {

  • {`${index + 1}. ${task.title || 'Untitled task'}`}

    - {task.runner === 'agent' - ? `Runs with the agent ${task.agent_name || 'chosen for it'}.` - : 'Runs with the default model.'} + {task.merge + ? mergeTaskText(task) + : task.runner === 'agent' + ? `Runs with the agent ${task.agent_name || 'chosen for it'}.` + : 'Runs with the default model.'}

    {task.requested_actions.length ? (

    {`Needs: ${actionList(task.requested_actions)}.`}

    ) : null} - {task.action_kinds.length ? ( + {task.action_kinds.length && !task.merge ? (

    {`The agent can use: ${actionList(task.action_kinds)}.`}

    ) : null} {task.inputs.length ? ( -

    {`Reads: ${task.inputs.join(', ')}.`}

    +

    + {`${task.merge ? 'Files to merge, in order' : 'Reads'}: ${task.inputs.join(', ')}.`} +

    ) : null}
    diff --git a/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx b/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx index ca9fe8584..3bdbe6f75 100644 --- a/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx @@ -269,7 +269,7 @@ export function WorkflowFileSyncFields({ checked={config.use_changed_documents} disabled={disabled || !active} onChange={(checked) => update({ use_changed_documents: checked })} - description="Analyze tasks without selected documents run on the files this sync changed." + description="Analyze tasks without selected documents, and Merge files tasks set to merge the files File Sync changed, run on the files this sync changed." /> ); diff --git a/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx b/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx index 819ecaec7..ceb9b0a6e 100644 --- a/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx @@ -21,14 +21,24 @@ import { } from '../../lib/workflowFlow'; import { comparisonActionFromSelection, + cleanupWorkflowMergeOptions, documentActionFromSelection, + formatWorkflowMergeColumnAliases, findWorkflowAgent, isWorkflowPublicationCompletionPolicy, isWorkflowPublicationSourceKind, + parseWorkflowMergeColumnAliases, safeWorkflowAlias, savedOutputPublicationFormats, + workflowMergeActionFromSelection, + workflowMergeDocumentLimit, + workflowMergeEnabled, + workflowMergeValidationErrors, workflowFileSyncProvidesAnalyzeTargets, workflowInputProcessingErrors, + WORKFLOW_MERGE_KIND_INPUTS, + WORKFLOW_MERGE_KIND_LABELS, + WORKFLOW_MERGE_KINDS_AVAILABLE, WORKFLOW_APPROVAL_MESSAGE_LIMIT, workflowSchemaErrors, workflowAgentKey, @@ -43,6 +53,12 @@ import { type WorkflowAgentReference, type WorkflowInputBinding, type WorkflowInputOutput, + type WorkflowMergeActionOptions, + type WorkflowMergeKind, + type WorkflowMergeOptions, + type WorkflowMergeOutputFormat, + type WorkflowMergeSortRule, + type WorkflowMergeTargetMode, type WorkflowOutputContract, type WorkflowOutputKind, type WorkflowScope, @@ -746,6 +762,9 @@ function actionMode(action: WorkflowDocumentAction | undefined): string { if (action.type === 'comparison') { return 'comparison'; } + if (action.type === 'merge') { + return 'merge'; + } if (action.type === 'search') { return Array.isArray(action.document_ids) && action.document_ids.length ? 'search_selected' @@ -766,16 +785,463 @@ function splitComparisonEvidence( return { left, right }; } +function mergeKind(action: WorkflowDocumentAction | undefined): WorkflowMergeKind { + return ['tabular', 'workbook', 'pdf', 'docx', 'pptx'].includes(String(action?.merge_kind)) + ? action?.merge_kind as WorkflowMergeKind : 'tabular'; +} + +function mergeTargetMode(action: WorkflowDocumentAction | undefined): WorkflowMergeTargetMode { + return ['selected', 'all', 'recent', 'changed'].includes(String(action?.target_mode)) + ? action?.target_mode as WorkflowMergeTargetMode : 'selected'; +} + +function mergeOutputFormat(action: WorkflowDocumentAction | undefined, kind: WorkflowMergeKind): WorkflowMergeOutputFormat { + if (kind === 'tabular' && (action?.output_format === 'xlsx' || action?.output_format === 'csv')) { + return action.output_format; + } + if (kind === 'tabular') { + return 'csv'; + } + return kind === 'workbook' ? 'xlsx' : kind; +} + +function mergeOptions(action: WorkflowDocumentAction | undefined): WorkflowMergeOptions { + return isRecord(action?.merge_options) ? { ...action.merge_options as WorkflowMergeOptions } : {}; +} + +function optionTextList(value: unknown): string { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string').join('\n') : ''; +} + +function setOptionTextList(options: WorkflowMergeOptions, key: keyof WorkflowMergeOptions, value: string): WorkflowMergeOptions { + const next = { ...options }; + const values = value.split(/\r?\n|,/).map((item) => item.trim()).filter(Boolean); + if (values.length) { + next[key] = values as never; + } else { + delete next[key]; + } + return next; +} + +function sortRules(value: unknown): WorkflowMergeSortRule[] { + const rules = Array.isArray(value) ? value.filter(isRecord).slice(0, 3) : []; + return [0, 1, 2].map((index) => { + const rule = rules[index]; + return { + column: typeof rule?.column === 'string' ? rule.column : '', + descending: rule?.descending === true, + value_type: ['text', 'number', 'date'].includes(String(rule?.value_type)) + ? rule?.value_type : 'text', + } as WorkflowMergeSortRule; + }); +} + +function updateSortRule(options: WorkflowMergeOptions, index: number, patch: Partial): WorkflowMergeOptions { + const rules = sortRules(options.sort_by).map((rule, position) => (position === index ? { ...rule, ...patch } : rule)) + .filter((rule) => rule.column.trim()); + const next = { ...options }; + if (rules.length) next.sort_by = rules; + else delete next.sort_by; + return next; +} + +function moveReference(references: WorkflowReferenceInput[], index: number, direction: -1 | 1): WorkflowReferenceInput[] { + const next = references.slice(); + const target = index + direction; + if (target < 0 || target >= references.length) { + return references; + } + [next[index], next[target]] = [next[target], next[index]]; + return next; +} + +function MergeActionFields({ + action, + evidence, + changedFileTargets, + maxSelectedDocuments, + onActionChange, + onEvidenceChange, +}: { + action: WorkflowDocumentAction | undefined; + evidence: WorkflowReferenceInput[]; + changedFileTargets: boolean; + maxSelectedDocuments?: number; + onActionChange: (updates: { + mergeKind?: WorkflowMergeKind; + targetMode?: WorkflowMergeTargetMode; + outputFormat?: WorkflowMergeOutputFormat; + recentWindowMinutes?: number; + outputFileName?: string; + mergeOptions?: WorkflowMergeOptions; + }) => void; + onEvidenceChange: (references: WorkflowReferenceInput[]) => void; +}) { + const kind = mergeKind(action); + const targetMode = mergeTargetMode(action); + const options = mergeOptions(action); + const [aliasesText, setAliasesText] = useState(() => formatWorkflowMergeColumnAliases(options.column_aliases)); + const aliasParse = parseWorkflowMergeColumnAliases(aliasesText); + const validationErrors = workflowMergeValidationErrors(action, { + isFileSyncWorkflow: changedFileTargets, + maxSelectedDocuments, + }); + const kindOptions = WORKFLOW_MERGE_KINDS_AVAILABLE.includes(kind) + ? WORKFLOW_MERGE_KINDS_AVAILABLE + : [kind, ...WORKFLOW_MERGE_KINDS_AVAILABLE]; + const applyOptions = (nextOptions: WorkflowMergeOptions) => { + onActionChange({ mergeOptions: cleanupWorkflowMergeOptions(kind, nextOptions) ?? nextOptions }); + }; + const applyAliases = (value: string) => { + setAliasesText(value); + const parsed = parseWorkflowMergeColumnAliases(value); + if (!parsed.errors.length) { + const next = { ...options }; + if (Object.keys(parsed.aliases).length) next.column_aliases = parsed.aliases; + else delete next.column_aliases; + applyOptions(next); + } + }; + const sheetMode = options.sheet ? 'named' : options.sheets === 'all' ? 'all' : 'first'; + return ( +
    +
    + + +
    + {targetMode === 'recent' ? ( + + ) : null} +
    + {kind === 'tabular' ? ( + + ) : ( +

    + Output format: {mergeOutputFormat(action, kind).toUpperCase()} +

    + )} + +
    + {targetMode === 'selected' && evidence.length ? ( +
    +

    Merge order

    +
      + {evidence.map((item, index) => ( +
    1. + {index + 1}. {item.name} + onEvidenceChange(moveReference(evidence, index, -1))} + aria-label={`Move ${item.name} up in merge order`}> + Up + + onEvidenceChange(moveReference(evidence, index, 1))} + aria-label={`Move ${item.name} down in merge order`}> + Down + +
      2. + ))} +
    +
    + ) : null} +
    + More merge options +
    + {kind === 'tabular' ? ( + <> + + {options.schema_policy === 'mapped' ? ( +