diff --git a/application/single_app/admin_settings_fields.py b/application/single_app/admin_settings_fields.py index 95d0dec11..923af7868 100644 --- a/application/single_app/admin_settings_fields.py +++ b/application/single_app/admin_settings_fields.py @@ -310,6 +310,13 @@ DOCUMENT_ACTION_WORKFLOW_MIN_LIMIT = 2 DOCUMENT_ACTION_WORKFLOW_MAX_LIMIT = 1000 DOCUMENT_ACTION_WORKFLOW_DEFAULT_LIMIT = 10 +# Merge, mirrored from MERGE_* in functions_document_actions.py. +DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_LIMIT = 10 +DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_LIMIT = 100 +DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT = 1000 +DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT = 1_000_000 +DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_ROWS = 250_000 +DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_ROWS = 1_000_000 # Schemes permitted for administrator-configured navigation links. These render # into an anchor href, so allowing arbitrary schemes would let a saved link @@ -3820,6 +3827,9 @@ {"value": "document_analyze", "label": "Analyse documents"}, {"value": "document_compare", "label": "Compare documents"}, {"value": "tabular_analyze", "label": "Analyse spreadsheets"}, + {"value": "tabular_inspect", "label": "Inspect spreadsheets"}, + {"value": "tabular_merge", "label": "Merge spreadsheets"}, + {"value": "document_merge", "label": "Merge documents"}, {"value": "web_search", "label": "Search the web"}, {"value": "url_fetch", "label": "Read linked pages"}, {"value": "deep_research", "label": "Research in depth"}, @@ -4619,6 +4629,74 @@ "group": "Comparison", "depends_on": {"key": "document_action_comparison_enabled", "equals": True}, }, + { + "key": "document_action_merge_enabled", + "settings_path": ["document_action_capabilities", "merge", "enabled"], + "type": "switch", + "label": "Enable Merge", + "help": ( + "Lets chat orchestration combine the rows of several CSV or Excel files " + "into one table, lining up columns that differ when asked, then deliver it " + "as a CSV or Excel file. Rows are appended exactly by code, not rewritten by " + "a model. Also allows inspecting the files' columns first." + ), + "default": True, + "group": "Merge", + }, + { + "key": "document_action_merge_chat_max_documents", + "settings_path": ["document_action_capabilities", "merge", "chat_max_documents"], + "type": "number", + "label": "Merge: Chat Document Limit", + "help": ( + "Most files one chat request may merge or inspect. Merging is cheaper than " + "Analyze because no model reads the files, so this can be higher." + ), + "default": DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_LIMIT, + "min": DOCUMENT_ACTION_CHAT_MIN_LIMIT, + "max": DOCUMENT_ACTION_CHAT_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_workflow_max_documents", + "settings_path": ["document_action_capabilities", "merge", "workflow_max_documents"], + "type": "number", + "label": "Merge: Workflow Document Limit", + "help": "The same limit for a workflow run, where large batches belong.", + "default": DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_LIMIT, + "min": DOCUMENT_ACTION_WORKFLOW_MIN_LIMIT, + "max": DOCUMENT_ACTION_WORKFLOW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_chat_max_rows", + "settings_path": ["document_action_capabilities", "merge", "chat_max_rows"], + "type": "number", + "label": "Merge: Chat Row Limit", + "help": ( + "Most rows one chat merge may produce. A few very large files can cost more " + "than many small ones, so rows are limited as well as files." + ), + "default": DOCUMENT_ACTION_MERGE_CHAT_DEFAULT_ROWS, + "min": DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT, + "max": DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, + { + "key": "document_action_merge_workflow_max_rows", + "settings_path": ["document_action_capabilities", "merge", "workflow_max_rows"], + "type": "number", + "label": "Merge: Workflow Row Limit", + "help": "The same row limit for a workflow run.", + "default": DOCUMENT_ACTION_MERGE_WORKFLOW_DEFAULT_ROWS, + "min": DOCUMENT_ACTION_MERGE_ROW_MIN_LIMIT, + "max": DOCUMENT_ACTION_MERGE_ROW_MAX_LIMIT, + "group": "Merge", + "depends_on": {"key": "document_action_merge_enabled", "equals": True}, + }, ], # Rendered by V1 only while Workspace Mode is on, matching agent-toggles-card. "plugin-feature-toggles": [ @@ -7123,7 +7201,7 @@ def _apply_nested_paths(normalized, current_settings, warnings=None): Deep Research reading the stale copy. ``settings_path`` names a single destination as a list of segments. - ``document_action_capabilities`` holds six values across two action types, + ``document_action_capabilities`` holds the Analyze, Comparison and Merge values, and nothing reads a flattened form of them, so writing the flat keys through would save a setting the application never looks at. diff --git a/application/single_app/config.py b/application/single_app/config.py index d3344236a..301f950dd 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -101,7 +101,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.238" +VERSION = "0.261.248" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_document_actions.py b/application/single_app/functions_document_actions.py index aa6652ff8..3878772f1 100644 --- a/application/single_app/functions_document_actions.py +++ b/application/single_app/functions_document_actions.py @@ -16,6 +16,7 @@ DOCUMENT_ACTION_TYPE_SEARCH = 'search' DOCUMENT_ACTION_TYPE_ANALYZE = 'analyze' DOCUMENT_ACTION_TYPE_COMPARISON = 'comparison' +DOCUMENT_ACTION_TYPE_MERGE = 'merge' DOCUMENT_ACTION_ANALYSIS_MODE_COMBINED = 'combined' DOCUMENT_ACTION_ANALYSIS_MODE_PER_DOCUMENT = 'per_document' DOCUMENT_ACTION_TARGET_MODE_ALL = 'all' @@ -38,6 +39,7 @@ DOCUMENT_ACTION_TYPE_SEARCH, DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON, + DOCUMENT_ACTION_TYPE_MERGE, } DOCUMENT_ACTION_LIMIT_BOUNDS = { DOCUMENT_ACTION_CONTEXT_CHAT: { @@ -49,6 +51,13 @@ 'max': 1000, }, } +# Merging is bounded by rows as well as files: a few very large files can cost more than +# many small ones. CSV exports allow at most 1,000,000 records. +MERGE_ROW_LIMIT_BOUNDS = {'min': 1000, 'max': 1_000_000} +MERGE_CHAT_DEFAULT_MAX_DOCUMENTS = 10 +MERGE_WORKFLOW_DEFAULT_MAX_DOCUMENTS = 100 +MERGE_CHAT_DEFAULT_MAX_ROWS = 250_000 +MERGE_WORKFLOW_DEFAULT_MAX_ROWS = 1_000_000 DEFAULT_DOCUMENT_ACTION_CAPABILITIES = { DOCUMENT_ACTION_TYPE_ANALYZE: { 'enabled': True, @@ -60,7 +69,84 @@ 'chat_max_documents': CHAT_DOCUMENT_ANALYSIS_MAX_DOCUMENTS, 'workflow_max_documents': WORKFLOW_DOCUMENT_ANALYSIS_MAX_DOCUMENTS, }, + DOCUMENT_ACTION_TYPE_MERGE: { + 'enabled': True, + 'chat_max_documents': MERGE_CHAT_DEFAULT_MAX_DOCUMENTS, + 'workflow_max_documents': MERGE_WORKFLOW_DEFAULT_MAX_DOCUMENTS, + 'chat_max_rows': MERGE_CHAT_DEFAULT_MAX_ROWS, + 'workflow_max_rows': MERGE_WORKFLOW_DEFAULT_MAX_ROWS, + }, } +# Classic chat and its document-action payloads only know Analyze and Comparison. Merge is +# reached through V2 orchestration and workflow tasks, so it is never offered to a caller +# that does not ask for it explicitly. +DOCUMENT_ACTION_TYPES_WITH_PAYLOAD_SUPPORT = ( + DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_COMPARISON, +) + +# A workflow Merge task combines many files into one output file. The kind decides which +# files it reads and what it produces; a kind missing from MERGE_KINDS_AVAILABLE is refused. +MERGE_KIND_TABULAR = 'tabular' +MERGE_KIND_WORKBOOK = 'workbook' +MERGE_KIND_PDF = 'pdf' +MERGE_KIND_DOCX = 'docx' +MERGE_KIND_PPTX = 'pptx' +MERGE_KINDS = (MERGE_KIND_TABULAR, MERGE_KIND_WORKBOOK, MERGE_KIND_PDF, MERGE_KIND_DOCX, MERGE_KIND_PPTX) +MERGE_KINDS_AVAILABLE = MERGE_KINDS +MERGE_KIND_LABELS = { + MERGE_KIND_TABULAR: 'spreadsheet', + MERGE_KIND_WORKBOOK: 'workbook', + MERGE_KIND_PDF: 'PDF', + MERGE_KIND_DOCX: 'Word', + MERGE_KIND_PPTX: 'PowerPoint', +} +MERGE_KIND_EXTENSIONS = { + MERGE_KIND_TABULAR: ('.csv', '.xlsx', '.xlsm', '.xls'), + MERGE_KIND_WORKBOOK: ('.csv', '.xlsx', '.xlsm', '.xls'), + MERGE_KIND_PDF: ('.pdf',), + MERGE_KIND_DOCX: ('.docx',), + MERGE_KIND_PPTX: ('.pptx',), +} +MERGE_KIND_OUTPUT_FORMATS = { + MERGE_KIND_TABULAR: ('csv', 'xlsx'), + MERGE_KIND_WORKBOOK: ('xlsx',), + MERGE_KIND_PDF: ('pdf',), + MERGE_KIND_DOCX: ('docx',), + MERGE_KIND_PPTX: ('pptx',), +} +MERGE_KIND_OPTION_KEYS = { + MERGE_KIND_TABULAR: ( + 'schema_policy', 'columns', 'column_aliases', 'sheet', 'sheets', 'header_row', + 'include_source_column', 'source_column_name', 'on_incompatible', 'dedupe', + 'dedupe_columns', 'dedupe_keep', 'sort_by', + ), + MERGE_KIND_WORKBOOK: ('sheets', 'sheet'), + MERGE_KIND_PDF: ('bookmarks',), + MERGE_KIND_DOCX: ('formatting', 'page_breaks', 'source_headings'), + MERGE_KIND_PPTX: ('formatting', 'sections'), +} +# Files are named by the user ("Changed", "All"...) or chosen from the workspace at run time. +MERGE_TARGET_MODE_CHANGED = 'changed' +MERGE_TARGET_MODES = ( + DOCUMENT_ACTION_TARGET_MODE_SELECTED, + DOCUMENT_ACTION_TARGET_MODE_ALL, + DOCUMENT_ACTION_TARGET_MODE_RECENT, + MERGE_TARGET_MODE_CHANGED, +) +MERGE_DEFAULT_RECENT_WINDOW_MINUTES = 60 +MERGE_OUTPUT_FILE_NAME_MAX_CHARS = 100 +_MERGE_FILE_NAME_FORBIDDEN = frozenset('/\\:*?"<>|') +_MERGE_DOCUMENT_FORMATTING = ('keep_source', 'use_first') +_MERGE_SHEET_MODES = ('first', 'all') + + +class MergeActionError(ValueError): + """A reviewed reason a Merge action is invalid; its text never repeats the caller's input. + + Workflow saves show it as written. Spreadsheet option errors from the merge engine stay plain + ``ValueError``s, because they can quote column names. + """ def normalize_document_action_type(action_type): @@ -108,6 +194,14 @@ def _coerce_document_action_limit(value, default_value, execution_context): return max(bounds['min'], min(bounds['max'], normalized_value)) +def _coerce_merge_row_limit(value, default_value): + try: + normalized_value = int(value) + except (TypeError, ValueError): + normalized_value = int(default_value) + return max(MERGE_ROW_LIMIT_BOUNDS['min'], min(MERGE_ROW_LIMIT_BOUNDS['max'], normalized_value)) + + def get_default_document_action_capabilities(): return copy.deepcopy(DEFAULT_DOCUMENT_ACTION_CAPABILITIES) @@ -138,6 +232,15 @@ def normalize_document_action_capabilities(settings_or_capabilities=None): DOCUMENT_ACTION_CONTEXT_WORKFLOW, ), } + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + normalized_capabilities[action_type].update({ + 'chat_max_rows': _coerce_merge_row_limit( + raw_capability.get('chat_max_rows'), default_capability['chat_max_rows'], + ), + 'workflow_max_rows': _coerce_merge_row_limit( + raw_capability.get('workflow_max_rows'), default_capability['workflow_max_rows'], + ), + }) return normalized_capabilities @@ -158,14 +261,32 @@ def is_document_action_enabled(action_type, settings=None): return bool(capability.get('enabled', False)) -def get_enabled_document_action_types(settings=None): +def get_enabled_document_action_types(settings=None, *, include_merge=False): + """Enabled action types for a payload owner. + + ``include_merge`` is opt-in so callers whose payloads cannot carry a merge action + (classic chat) never accept one. + """ enabled_action_types = {DOCUMENT_ACTION_TYPE_NONE, DOCUMENT_ACTION_TYPE_SEARCH} - for action_type in (DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON): + action_types = DOCUMENT_ACTION_TYPES_WITH_PAYLOAD_SUPPORT + ( + (DOCUMENT_ACTION_TYPE_MERGE,) if include_merge else () + ) + for action_type in action_types: if is_document_action_enabled(action_type, settings=settings): enabled_action_types.add(action_type) return enabled_action_types +def get_document_action_max_rows(action_type, execution_context, settings=None): + """The merged-row ceiling for Merge in chat or workflows; None for other actions.""" + if normalize_document_action_type(action_type) != DOCUMENT_ACTION_TYPE_MERGE: + return None + capability = get_document_action_capability(DOCUMENT_ACTION_TYPE_MERGE, settings=settings) + field_name = 'workflow_max_rows' if execution_context == DOCUMENT_ACTION_CONTEXT_WORKFLOW else 'chat_max_rows' + default_value = DEFAULT_DOCUMENT_ACTION_CAPABILITIES[DOCUMENT_ACTION_TYPE_MERGE][field_name] + return _coerce_merge_row_limit(capability.get(field_name, default_value), default_value) + + def get_document_action_max_documents(action_type, execution_context, settings=None): normalized_action_type = normalize_document_action_type(action_type) capability_action_type = DOCUMENT_ACTION_TYPE_ANALYZE if normalized_action_type == DOCUMENT_ACTION_TYPE_SEARCH else normalized_action_type @@ -193,6 +314,11 @@ def get_document_action_max_documents_by_type(execution_context, settings=None): execution_context, settings=settings, ), + DOCUMENT_ACTION_TYPE_MERGE: get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, + execution_context, + settings=settings, + ), } @@ -209,9 +335,121 @@ def _build_document_action_disabled_message(action_type): return 'Document analysis is currently disabled in admin settings.' if action_type == DOCUMENT_ACTION_TYPE_COMPARISON: return 'Document comparison is currently disabled in admin settings.' + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + return 'File merging is turned off in admin settings, or is not available here.' return 'The selected document action is currently disabled in admin settings.' +def normalize_merge_output_file_name(value, output_format): + """A safe base name for a merged file without its extension, or '' when none is set.""" + if value is None or value == '': + return '' + if not isinstance(value, str): + raise MergeActionError('The merged file name must be text.') + name = ' '.join(value.split()) + suffix = f'.{output_format}' + if name.lower().endswith(suffix): + name = name[:-len(suffix)].rstrip() + if ( + len(name) > MERGE_OUTPUT_FILE_NAME_MAX_CHARS or name.startswith('.') + or any(character in _MERGE_FILE_NAME_FORBIDDEN or ord(character) < 32 for character in name) + ): + raise MergeActionError( + f'The merged file name must be at most {MERGE_OUTPUT_FILE_NAME_MAX_CHARS} characters, ' + 'must not start with a dot, and must not contain / \\ : * ? " < > |.' + ) + return name + + +def _normalize_merge_options(kind, options): + """Kind-specific merge options with unset values dropped; rules match chat merges.""" + if options is None: + return {} + if not isinstance(options, dict): + raise MergeActionError('Merge options must be an object.') + if set(options) - set(MERGE_KIND_OPTION_KEYS[kind]): + raise MergeActionError( + f"Some merge options don't apply to {MERGE_KIND_LABELS[kind]} merges. Remove them." + ) + cleaned = {key: copy.deepcopy(value) for key, value in options.items() if value is not None} + if kind == MERGE_KIND_TABULAR: + # The merge engine imports only the standard library and owns the spreadsheet rules. + from functions_tabular_merge import TabularMergeError, tabular_merge_options_from_arguments + + try: + tabular_merge_options_from_arguments(cleaned) + except TabularMergeError as exc: + raise ValueError(exc.message) from exc + return cleaned + for key in ('bookmarks', 'sections', 'page_breaks', 'source_headings'): + if key in cleaned and type(cleaned[key]) is not bool: + raise MergeActionError(f'The {key} merge option must be true or false.') + if 'formatting' in cleaned and cleaned['formatting'] not in _MERGE_DOCUMENT_FORMATTING: + raise MergeActionError('Formatting must be keep_source or use_first.') + if 'sheets' in cleaned and cleaned['sheets'] not in _MERGE_SHEET_MODES: + raise MergeActionError('Sheets must be first or all.') + if 'sheet' in cleaned: + if not isinstance(cleaned['sheet'], str) or not cleaned['sheet'].strip() or len(cleaned['sheet']) > 31: + raise MergeActionError('A sheet name must be 1 to 31 characters.') + if cleaned.get('sheets') == 'all': + raise MergeActionError('Name one sheet, or read all sheets, but not both.') + return cleaned + + +def _normalize_merge_action(source_action, normalized_action, max_documents=None): + """A workflow Merge task: which files, in which order, into which kind of file.""" + kind = str(source_action.get('merge_kind') or MERGE_KIND_TABULAR).strip().lower() + if kind not in MERGE_KINDS: + raise MergeActionError(f"Merge kind must be one of: {', '.join(MERGE_KINDS)}.") + if kind not in MERGE_KINDS_AVAILABLE: + raise MergeActionError(f'Merging {MERGE_KIND_LABELS[kind]} files is not available yet.') + target_mode = str(source_action.get('target_mode') or DOCUMENT_ACTION_TARGET_MODE_SELECTED).strip().lower() + if target_mode not in MERGE_TARGET_MODES: + raise MergeActionError(f"Merge files must be one of: {', '.join(MERGE_TARGET_MODES)}.") + doc_scope = str(source_action.get('doc_scope') or 'all').strip().lower() + if doc_scope not in {'all', 'personal', 'group', 'public'}: + raise MergeActionError('The merge workspace scope must be all, personal, group or public.') + resolved = bool(source_action.get('merge_targets_resolved')) + document_ids = normalize_search_id_list(source_action.get('document_ids')) + if target_mode == DOCUMENT_ACTION_TARGET_MODE_SELECTED: + if len(document_ids) < 2: + raise MergeActionError('Select at least two files to merge.') + if max_documents is not None and len(document_ids) > max_documents: + raise MergeActionError(f'A merge supports up to {max_documents} files at a time.') + elif not resolved: + # Files found at run time are never stored with the task. + document_ids = [] + # Files a run found are counted by the merge itself, which names the count and the limit. + output_formats = MERGE_KIND_OUTPUT_FORMATS[kind] + output_format = str(source_action.get('output_format') or output_formats[0]).strip().lower() + if output_format not in output_formats: + raise MergeActionError( + f"A {MERGE_KIND_LABELS[kind]} merge creates " + f"{' or '.join(value.upper() for value in output_formats)} files." + ) + normalized_action.update({ + 'merge_kind': kind, + 'target_mode': target_mode, + 'doc_scope': doc_scope, + 'active_group_ids': normalize_search_id_list(source_action.get('active_group_ids')), + 'active_public_workspace_id': normalize_search_id_list(source_action.get('active_public_workspace_id')), + 'document_ids': document_ids, + 'output_format': output_format, + 'merge_options': _normalize_merge_options(kind, source_action.get('merge_options')), + }) + if target_mode == DOCUMENT_ACTION_TARGET_MODE_RECENT: + window = source_action.get('recent_window_minutes') + normalized_action['recent_window_minutes'] = normalize_recent_document_window_minutes( + MERGE_DEFAULT_RECENT_WINDOW_MINUTES if window is None else window, + ) + file_name = normalize_merge_output_file_name(source_action.get('output_file_name'), output_format) + if file_name: + normalized_action['output_file_name'] = file_name + if resolved: + normalized_action['merge_targets_resolved'] = True + return normalized_action + + def _build_analyze_action(legacy_analyze=None): legacy_analyze = legacy_analyze if isinstance(legacy_analyze, dict) else {} if not legacy_analyze.get('enabled'): @@ -302,7 +540,11 @@ def normalize_document_action_config( } normalized_allowed_action_types.add(DOCUMENT_ACTION_TYPE_NONE) if action_type not in normalized_allowed_action_types: - raise ValueError(_build_document_action_disabled_message(action_type)) + message = _build_document_action_disabled_message(action_type) + raise MergeActionError(message) if action_type == DOCUMENT_ACTION_TYPE_MERGE else ValueError(message) + + if action_type == DOCUMENT_ACTION_TYPE_MERGE: + return _normalize_merge_action(source_action, normalized_action, resolved_max_documents) if action_type == DOCUMENT_ACTION_TYPE_ANALYZE: if 'analysis_options' in source_action or 'transformation_spec' in source_action: diff --git a/application/single_app/functions_document_merge.py b/application/single_app/functions_document_merge.py new file mode 100644 index 000000000..f75d7bd21 --- /dev/null +++ b/application/single_app/functions_document_merge.py @@ -0,0 +1,140 @@ +# functions_document_merge.py +"""Ordered assembly of several PDF, Word, PowerPoint or spreadsheet files into one file. + +Version: 0.261.248 + +The engine is pure: it receives already-authorized byte loaders, never resolves +documents, settings, storage or routes, and performs no model work. Each assembler reads +one source at a time, checks a package's size and refuses XML parts that declare a document +type or aren't UTF-8 or UTF-16 before parsing it, and writes the output to a bounded spooled +file whose size is checked before it is returned; a spool larger than 16 MiB spills into the +container's scratch directory (0.261.247). The same files and options always produce the +same bytes, so a replayed merge reuses its file. CodeQL import-cycle cleanup added in +0.261.248. +""" + +from typing import Callable, Optional, Sequence + +from functions_document_merge_core import ( + DOCUMENT_MERGE_KINDS, + DOCUMENT_MERGE_OUTPUTS, + DOCUMENT_MERGE_REPORT_VERSION, + DOCUMENT_MERGE_SOURCE_EXTENSIONS, + DocumentMergeCancelled, + DocumentMergeError, + DocumentMergeLimits, + DocumentMergeOptions, + DocumentMergePart, + DocumentMergeResult, + FORMATTING_KEEP_SOURCE, + FORMATTING_USE_FIRST, + MERGE_KIND_DOCX, + MERGE_KIND_PDF, + MERGE_KIND_PPTX, + MERGE_KIND_WORKBOOK, + MergeContext, + WORKBOOK_SHEETS_ALL, + WORKBOOK_SHEETS_FIRST, +) + + +def merge_documents( + kind, + parts: Sequence[DocumentMergePart], + *, + options: Optional[DocumentMergeOptions] = None, + limits: Optional[DocumentMergeLimits] = None, + cancel_requested: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[dict], None]] = None, + min_parts: int = 2, +) -> DocumentMergeResult: + """Assemble the parts, in order, into one file of the given kind. + + ``min_parts`` is 1 only for a scheduled merge of whatever files arrived, which may be one. + """ + if kind not in DOCUMENT_MERGE_KINDS: + raise DocumentMergeError("unsupported_format", "This kind of file can't be merged.") + if min_parts not in (1, 2): + raise ValueError("min_parts must be 1 or 2.") + options = options or DocumentMergeOptions() + limits = limits or DocumentMergeLimits() + parts = list(parts or ()) + _validate_parts(kind, parts, limits, min_parts) + context = MergeContext(kind, parts, options, limits, cancel_requested, on_progress) + # Each assembler loads only when its kind is merged; the Office stacks are large. + if kind == MERGE_KIND_PDF: + from functions_document_merge_pdf import assemble_pdf as assemble + elif kind == MERGE_KIND_DOCX: + from functions_document_merge_docx import assemble_docx as assemble + elif kind == MERGE_KIND_PPTX: + from functions_document_merge_pptx import assemble_pptx as assemble + else: + from functions_document_merge_workbook import assemble_workbook as assemble + try: + return assemble(context) + except DocumentMergeError as error: + context.report["status"] = "failed" + if error.report is None: + error.report = context.report + raise + + +def _validate_parts(kind, parts, limits, min_parts=2): + if len(parts) < min_parts: + raise DocumentMergeError("too_few_sources", "Merging needs at least two files.") + if len(parts) > limits.max_parts: + raise DocumentMergeError( + "too_many_sources", f"{len(parts)} files were selected; at most {limits.max_parts} can be merged here.", + ) + allowed = DOCUMENT_MERGE_SOURCE_EXTENSIONS[kind] + seen = set() + for part in parts: + if not isinstance(part, DocumentMergePart): + raise DocumentMergeError("invalid_source", "Each merge source must be a DocumentMergePart.") + if not isinstance(part.source_id, str) or not part.source_id.strip(): + raise DocumentMergeError("invalid_source", "Each merge source needs an identifier.") + if part.source_id in seen: + raise DocumentMergeError("duplicate_source", "The same file was selected more than once.") + seen.add(part.source_id) + if not callable(part.load_bytes): + raise DocumentMergeError("invalid_source", "Each merge source needs a byte loader.") + if part.resolved_extension() not in allowed: + raise DocumentMergeError( + "unsupported_format", + f"{part.display_name()} can't be merged into a {kind.upper() if kind != MERGE_KIND_WORKBOOK else 'workbook'} " + f"file. Allowed: {', '.join(allowed)}.", + ) + if part.pages is not None and kind not in (MERGE_KIND_PDF, MERGE_KIND_PPTX): + raise DocumentMergeError("range_invalid", "Page ranges apply only to PDF and PowerPoint merges.") + + +def document_merge_kind_for_extension(extension): + """The merge kinds a file extension can take part in.""" + extension = str(extension or "").strip().lower() + if extension and not extension.startswith("."): + extension = f".{extension}" + return tuple(kind for kind in DOCUMENT_MERGE_KINDS if extension in DOCUMENT_MERGE_SOURCE_EXTENSIONS[kind]) + + +__all__ = [ + "DOCUMENT_MERGE_KINDS", + "DOCUMENT_MERGE_OUTPUTS", + "DOCUMENT_MERGE_REPORT_VERSION", + "DOCUMENT_MERGE_SOURCE_EXTENSIONS", + "DocumentMergeCancelled", + "DocumentMergeError", + "DocumentMergeLimits", + "DocumentMergeOptions", + "DocumentMergePart", + "DocumentMergeResult", + "FORMATTING_KEEP_SOURCE", + "FORMATTING_USE_FIRST", + "MERGE_KIND_DOCX", + "MERGE_KIND_PDF", + "MERGE_KIND_PPTX", + "MERGE_KIND_WORKBOOK", + "WORKBOOK_SHEETS_ALL", + "WORKBOOK_SHEETS_FIRST", + "document_merge_kind_for_extension", + "merge_documents", +] diff --git a/application/single_app/functions_document_merge_assembly.py b/application/single_app/functions_document_merge_assembly.py new file mode 100644 index 000000000..5b2297a55 --- /dev/null +++ b/application/single_app/functions_document_merge_assembly.py @@ -0,0 +1,209 @@ +# functions_document_merge_assembly.py +"""The retained description of a document merge, and its exact re-assembly for Render. + +Version: 0.261.245 +Implemented in: 0.261.245 + +A chat ``document_merge`` step assembles the authorized files once, to check them and +measure the result, and retains this description instead of the bytes: the kind, every +option, the files in order, and the merged file's size and SHA-256. ``render_file`` then +reads the same files again through a reader that the orchestration supplies and that +admits only the files the result was assembled from, assembles them again, and delivers +the file only when it is byte-for-byte the one that was checked. The assemblers are +deterministic, so it is unless a file changed. + +This module imports only the merge engine, so the export layer can use it. +""" + +import hashlib +import re + +from functions_document_merge import ( + DOCUMENT_MERGE_KINDS, + DOCUMENT_MERGE_OUTPUTS, + DocumentMergeError, + DocumentMergeLimits, + DocumentMergeOptions, + DocumentMergePart, + MERGE_KIND_DOCX, + MERGE_KIND_PDF, + MERGE_KIND_PPTX, + MERGE_KIND_WORKBOOK, + merge_documents, +) + + +DOCUMENT_ASSEMBLY_PROFILE = "document_assembly_v1" +# The file format each kind creates, as render_file names it. +ASSEMBLED_OUTPUT_FORMATS = { + MERGE_KIND_PDF: "pdf", + MERGE_KIND_DOCX: "docx", + MERGE_KIND_PPTX: "pptx", + MERGE_KIND_WORKBOOK: "xlsx", +} +ASSEMBLY_OPTION_FIELDS = ("formatting", "bookmarks", "sections", "page_breaks", "source_headings", "sheets", "sheet") +# The options each kind takes; the same as a workflow Merge task's. +ASSEMBLY_KIND_OPTION_FIELDS = { + MERGE_KIND_PDF: ("bookmarks",), + MERGE_KIND_DOCX: ("formatting", "page_breaks", "source_headings"), + MERGE_KIND_PPTX: ("formatting", "sections"), + MERGE_KIND_WORKBOOK: ("sheets", "sheet"), +} +MAX_ASSEMBLY_PARTS = 1000 +_DIGEST = re.compile(r"[a-f0-9]{64}\Z") +_MAX_IDENTIFIER = 256 +_MAX_FILE_NAME = 1024 + + +class DocumentAssemblyError(ValueError): + """A described merge that can't be delivered; the code is an export failure code.""" + + def __init__(self, code, message): + super().__init__(message) + self.code = code + self.message = message + + +def document_merge_options_from_arguments(arguments): + """The kind and engine options a merge step asks for; options of another kind are refused.""" + arguments = arguments if isinstance(arguments, dict) else {} + kind = arguments.get("kind") + if kind not in ASSEMBLY_KIND_OPTION_FIELDS: + raise DocumentMergeError("invalid_options", "Choose pdf, docx, pptx or workbook as the merge kind.") + present = {name for name in ASSEMBLY_OPTION_FIELDS if arguments.get(name) is not None} + if present - set(ASSEMBLY_KIND_OPTION_FIELDS[kind]): + raise DocumentMergeError("invalid_options", f"Some settings don't apply to a {kind} merge.") + if arguments.get("sheet") is not None and arguments.get("sheets") == "all": + raise DocumentMergeError("invalid_options", "Name one sheet, or read all sheets, but not both.") + return kind, DocumentMergeOptions(**{name: arguments[name] for name in present}) + + +def build_document_assembly(kind, options, parts, result, content_sha256): + """The description a document_merge step retains for the file it checked.""" + if kind not in DOCUMENT_MERGE_KINDS or not isinstance(options, DocumentMergeOptions): + raise DocumentAssemblyError("invalid_source", "The merge description is invalid.") + return validate_document_assembly({ + "profile": DOCUMENT_ASSEMBLY_PROFILE, + "kind": kind, + "options": {name: getattr(options, name) for name in ASSEMBLY_OPTION_FIELDS}, + "parts": [{"document_id": part.source_id, "file_name": part.display_name()} for part in parts], + "output": { + "format": ASSEMBLED_OUTPUT_FORMATS[kind], + "media_type": result.media_type, + "size_bytes": result.size_bytes, + "content_sha256": content_sha256, + }, + }) + + +def validate_document_assembly(value): + """A checked copy of a retained description; anything unexpected is refused, never repaired.""" + if not isinstance(value, dict) or set(value) != {"profile", "kind", "options", "parts", "output"}: + raise DocumentAssemblyError("invalid_source", "The merge description is invalid.") + kind = value["kind"] + if value["profile"] != DOCUMENT_ASSEMBLY_PROFILE or kind not in DOCUMENT_MERGE_KINDS: + raise DocumentAssemblyError("invalid_source", "The merge description is invalid.") + options = value["options"] + if not isinstance(options, dict) or set(options) != set(ASSEMBLY_OPTION_FIELDS): + raise DocumentAssemblyError("invalid_source", "The merge options are invalid.") + try: + DocumentMergeOptions(**options) + except (DocumentMergeError, TypeError) as exc: + raise DocumentAssemblyError("invalid_source", "The merge options are invalid.") from exc + parts = value["parts"] + if not isinstance(parts, list) or not 2 <= len(parts) <= MAX_ASSEMBLY_PARTS: + raise DocumentAssemblyError("invalid_source", "The merge description names an invalid set of files.") + seen = set() + for entry in parts: + if ( + not isinstance(entry, dict) or set(entry) != {"document_id", "file_name"} + or not isinstance(entry["document_id"], str) or not entry["document_id"].strip() + or len(entry["document_id"]) > _MAX_IDENTIFIER or entry["document_id"] in seen + or not isinstance(entry["file_name"], str) or len(entry["file_name"]) > _MAX_FILE_NAME + ): + raise DocumentAssemblyError("invalid_source", "The merge description names an invalid set of files.") + seen.add(entry["document_id"]) + output = value["output"] + if ( + not isinstance(output, dict) or set(output) != {"format", "media_type", "size_bytes", "content_sha256"} + or output["format"] != ASSEMBLED_OUTPUT_FORMATS[kind] + or output["media_type"] != DOCUMENT_MERGE_OUTPUTS[kind][1] + or type(output["size_bytes"]) is not int or output["size_bytes"] < 1 + or not isinstance(output["content_sha256"], str) or not _DIGEST.match(output["content_sha256"]) + ): + raise DocumentAssemblyError("invalid_source", "The merged file's description is invalid.") + return { + "profile": DOCUMENT_ASSEMBLY_PROFILE, + "kind": kind, + "options": dict(options), + "parts": [dict(entry) for entry in parts], + "output": dict(output), + } + + +def assembled_output_format(value): + """The format a described merge creates, after checking the description.""" + return validate_document_assembly(value)["output"]["format"] + + +def reassemble_document(value, read_document, *, max_output_bytes, check=None): + """Assemble the described files again; return the result only when it is byte-identical. + + ``read_document(document_id)`` returns one file's current bytes. ``check()`` raises to + stop; the engine calls it before every file is read and while the file is assembled. + Returns the merge result, which the caller closes, and its SHA-256. + """ + assembly = validate_document_assembly(value) + if not callable(read_document): + raise DocumentAssemblyError("unsupported_source", "A merged file needs its original files.") + if type(max_output_bytes) is not int or max_output_bytes < 1: + raise DocumentAssemblyError("invalid_limit", "A positive output limit is required.") + if assembly["output"]["size_bytes"] > max_output_bytes: + raise DocumentAssemblyError("size_limit", "The merged file is larger than this export allows.") + + def stop_requested(): + # check() raises to stop; the engine passes that failure through unchanged. + if check is not None: + check() + return False + + parts = [ + DocumentMergePart( + entry["document_id"], entry["file_name"], + lambda document_id=entry["document_id"]: read_document(document_id), + ) + for entry in assembly["parts"] + ] + limits = DocumentMergeLimits(max_parts=max(2, len(parts)), max_output_bytes=max_output_bytes) + try: + result = merge_documents( + assembly["kind"], parts, options=DocumentMergeOptions(**assembly["options"]), limits=limits, + cancel_requested=stop_requested, + ) + except DocumentMergeError as exc: + # The same files assembled once already within these limits; a refusal now means one changed. + raise DocumentAssemblyError("source_changed", "A merged file no longer matches its sources.") from exc + try: + digest = hashlib.sha256() + for chunk in result.iter_chunks(): + digest.update(chunk) + if result.size_bytes != assembly["output"]["size_bytes"] or digest.hexdigest() != assembly["output"]["content_sha256"]: + raise DocumentAssemblyError("source_changed", "A merged file no longer matches its sources.") + except BaseException: + result.close() + raise + return result, digest.hexdigest() + + +__all__ = [ + "ASSEMBLED_OUTPUT_FORMATS", + "ASSEMBLY_KIND_OPTION_FIELDS", + "ASSEMBLY_OPTION_FIELDS", + "DOCUMENT_ASSEMBLY_PROFILE", + "DocumentAssemblyError", + "assembled_output_format", + "build_document_assembly", + "document_merge_options_from_arguments", + "reassemble_document", + "validate_document_assembly", +] diff --git a/application/single_app/functions_document_merge_core.py b/application/single_app/functions_document_merge_core.py new file mode 100644 index 000000000..4f455aecb --- /dev/null +++ b/application/single_app/functions_document_merge_core.py @@ -0,0 +1,448 @@ +# functions_document_merge_core.py +"""Shared primitives for ordered document assembly. + +Version: 0.261.248 + +The engine is pure: it receives already-authorized byte loaders, never resolves +documents, settings, storage or routes, and performs no model work. Each assembler reads +one source at a time, checks a package's size and refuses XML parts that declare a document +type or aren't UTF-8 or UTF-16 before parsing it, and writes the output to a bounded spooled +file whose size is checked before it is returned; a spool larger than 16 MiB spills into the +container's scratch directory (0.261.247). The same files and options always produce the +same bytes, so a replayed merge reuses its file. +""" + +from dataclasses import dataclass, field +import io +import os +import tempfile +from typing import Callable, Optional, Tuple +import zipfile + +from functions_ooxml_package_guard import UnreadablePackageError, first_unsafe_xml_part +from functions_temp_files import scratch_file_dir + + +DOCUMENT_MERGE_REPORT_VERSION = "document-merge-report-v1" + +MERGE_KIND_PDF = "pdf" +MERGE_KIND_DOCX = "docx" +MERGE_KIND_PPTX = "pptx" +MERGE_KIND_WORKBOOK = "workbook" +# The kinds that have an assembler. +DOCUMENT_MERGE_KINDS = (MERGE_KIND_PDF, MERGE_KIND_DOCX, MERGE_KIND_PPTX, MERGE_KIND_WORKBOOK) + +DOCUMENT_MERGE_SOURCE_EXTENSIONS = { + MERGE_KIND_PDF: (".pdf",), + MERGE_KIND_DOCX: (".docx",), + MERGE_KIND_PPTX: (".pptx",), + MERGE_KIND_WORKBOOK: (".xlsx", ".xlsm", ".xls", ".csv"), +} +DOCUMENT_MERGE_OUTPUTS = { + MERGE_KIND_PDF: (".pdf", "application/pdf"), + MERGE_KIND_DOCX: (".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + MERGE_KIND_PPTX: (".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"), + MERGE_KIND_WORKBOOK: (".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), +} + +FORMATTING_KEEP_SOURCE = "keep_source" +FORMATTING_USE_FIRST = "use_first" +DOCUMENT_MERGE_FORMATTING = (FORMATTING_KEEP_SOURCE, FORMATTING_USE_FIRST) + +WORKBOOK_SHEETS_FIRST = "first" +WORKBOOK_SHEETS_ALL = "all" +WORKBOOK_SHEET_MODES = (WORKBOOK_SHEETS_FIRST, WORKBOOK_SHEETS_ALL) + +PART_STATUS_MERGED = "merged" +PART_STATUS_NOT_PROCESSED = "not_processed" + +MAX_PAGE_RANGES = 64 +_SPOOL_MEMORY_BYTES = 16 * 1024 * 1024 +_COPY_CHUNK_BYTES = 64 * 1024 +_PACKAGE_DATE_TIME = (2000, 1, 1, 0, 0, 0) +_ZIP_MAGIC = b"PK\x03\x04" +_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +_MIB = 1024 * 1024 + + +class DocumentMergeError(ValueError): + """A safe, user-presentable merge failure with a stable code.""" + + def __init__(self, code, message, *, report=None, details=None): + super().__init__(message) + self.code = code + self.message = message + self.report = report + self.details = dict(details or {}) + + +class DocumentMergeCancelled(DocumentMergeError): + """The caller asked the merge to stop.""" + + def __init__(self): + super().__init__("cancelled", "The merge was cancelled.") + + +@dataclass(frozen=True) +class DocumentMergePart: + """One authorized source and, for PDFs and decks, the 1-based inclusive ranges to keep.""" + + source_id: str + file_name: str + load_bytes: Callable[[], bytes] = field(repr=False) + extension: str = "" + pages: Optional[Tuple[Tuple[int, int], ...]] = None + + def resolved_extension(self): + extension = (self.extension or os.path.splitext(self.file_name or "")[1]).strip().lower() + if extension and not extension.startswith("."): + extension = f".{extension}" + return extension + + def display_name(self): + return str(self.file_name or "").strip() or "A selected file" + + +@dataclass(frozen=True) +class DocumentMergeOptions: + """Per-kind presentation choices; options a kind does not use are ignored.""" + + formatting: str = FORMATTING_KEEP_SOURCE + bookmarks: bool = True + sections: bool = True + page_breaks: bool = True + source_headings: bool = False + sheets: str = WORKBOOK_SHEETS_FIRST + sheet: Optional[str] = None + + def __post_init__(self): + if self.formatting not in DOCUMENT_MERGE_FORMATTING: + raise DocumentMergeError("invalid_options", "Formatting must be keep_source or use_first.") + if self.sheets not in WORKBOOK_SHEET_MODES: + raise DocumentMergeError("invalid_options", "Sheets must be first or all.") + for name in ("bookmarks", "sections", "page_breaks", "source_headings"): + if type(getattr(self, name)) is not bool: + raise DocumentMergeError("invalid_options", f"{name} must be true or false.") + if self.sheet is not None and (not isinstance(self.sheet, str) or not self.sheet.strip() or len(self.sheet) > 31): + raise DocumentMergeError("invalid_options", "A sheet name must be 1 to 31 characters.") + + +@dataclass(frozen=True) +class DocumentMergeLimits: + """Bounds for one merge; every value must be a positive integer.""" + + max_parts: int = 10 + max_source_bytes: int = 100 * _MIB + max_total_input_bytes: int = 300 * _MIB + max_output_bytes: int = 100 * _MIB + max_pages: int = 2000 + max_slides: int = 1000 + max_sheets: int = 100 + max_sheet_rows: int = 1_048_576 + max_total_cells: int = 10_000_000 + max_package_uncompressed_bytes: int = 1024 * _MIB + max_package_entries: int = 50_000 + + def __post_init__(self): + for name in self.__dataclass_fields__: + value = getattr(self, name) + if type(value) is not int or value < 1: + raise ValueError(f"{name} must be a positive integer.") + if self.max_parts < 2: + raise ValueError("max_parts must allow at least two parts.") + + +class DocumentMergeResult: + """The assembled file, its metadata and the merge report. Close it when done.""" + + def __init__(self, kind, spool, size_bytes, report): + self.kind = kind + self.file_extension, self.media_type = DOCUMENT_MERGE_OUTPUTS[kind] + self._spool = spool + self.size_bytes = size_bytes + self.report = report + + def read_bytes(self): + self._spool.seek(0) + return self._spool.read() + + def open_stream(self): + """The assembled file as a seekable stream at its start; it closes with the result.""" + self._spool.seek(0) + return self._spool + + def iter_chunks(self, chunk_size=1024 * 1024): + self._spool.seek(0) + while True: + chunk = self._spool.read(chunk_size) + if not chunk: + return + yield chunk + + def close(self): + self._spool.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + self.close() + return False + + +class MergeContext: + """Shared state for one assembler run: limits, report, cancellation and byte budget.""" + + def __init__(self, kind, parts, options, limits, cancel_requested, on_progress): + self.kind = kind + self.parts = parts + self.options = options + self.limits = limits + self._cancel_requested = cancel_requested + self._on_progress = on_progress + self._callback_error = None + self.input_bytes = 0 + self.report = { + "version": DOCUMENT_MERGE_REPORT_VERSION, + "kind": kind, + "status": "running", + "options": _option_summary(kind, options), + "parts": [_new_part_entry(part) for part in parts], + "totals": {"parts": len(parts), "pages": 0, "slides": 0, "sheets": 0, "bytes": 0}, + "warnings": [], + "limitations": [], + } + + def check_cancel(self): + if self._cancel_requested is None: + return + try: + requested = self._cancel_requested() + except Exception as exc: + # Remembered so an assembler never reports the caller's own failure as a bad file. + self._callback_error = exc + raise + if requested: + raise DocumentMergeCancelled() + + def blames_file(self, error): + """Whether an unexpected error inside an assembler may be reported as a problem with a file. + + A failing cancellation check and storage errors, such as a full temporary disk, come from + the caller or the host, so they pass through unchanged and the caller can retry. + """ + return error is not self._callback_error and not isinstance(error, OSError) + + def load(self, index): + """Read one part's bytes within the per-file and total input budgets.""" + self.check_cancel() + part = self.parts[index] + content = part.load_bytes() + if isinstance(content, bytearray): + content = bytes(content) + if not isinstance(content, bytes): + raise DocumentMergeError("source_unavailable", f"{part.display_name()} could not be read.") + if len(content) > self.limits.max_source_bytes: + raise DocumentMergeError( + "source_too_large", + f"{part.display_name()} is larger than the {self.limits.max_source_bytes // _MIB} MB merge limit.", + ) + self.input_bytes += len(content) + if self.input_bytes > self.limits.max_total_input_bytes: + raise DocumentMergeError("input_too_large", "The selected files are too large to merge together.") + return content + + def part_entry(self, index): + return self.report["parts"][index] + + def warn(self, code, message, index=None): + target = self.report["warnings"] if index is None else self.report["parts"][index]["warnings"] + warning = {"code": code, "message": message} + if warning not in target: + target.append(warning) + + def limit(self, message): + if message not in self.report["limitations"]: + self.report["limitations"].append(message) + + def progress(self, index): + entry = self.report["parts"][index] + entry["status"] = PART_STATUS_MERGED + if self._on_progress is not None: + self._on_progress({ + "index": index + 1, "total": len(self.parts), "file_name": entry["file_name"], + "pages": entry.get("pages", 0), "slides": entry.get("slides", 0), "sheets": entry.get("sheets", 0), + }) + + +def _option_summary(kind, options): + if kind == MERGE_KIND_PDF: + return {"bookmarks": options.bookmarks} + if kind == MERGE_KIND_DOCX: + return { + "formatting": options.formatting, "page_breaks": options.page_breaks, + "source_headings": options.source_headings, + } + if kind == MERGE_KIND_PPTX: + return {"formatting": options.formatting, "sections": options.sections} + return {"sheets": options.sheets, "sheet": options.sheet} + + +def _new_part_entry(part): + entry = { + "source_id": part.source_id, + "file_name": part.display_name(), + "status": PART_STATUS_NOT_PROCESSED, + "warnings": [], + } + if part.pages: + entry["ranges"] = [list(item) for item in part.pages] + return entry + + +def new_output_spool(): + # The container's dedicated scratch directory, or the platform default; never the working + # directory, which the container's non-root user can't write. + return tempfile.SpooledTemporaryFile(max_size=_SPOOL_MEMORY_BYTES, mode="w+b", dir=scratch_file_dir()) + + +def fixed_zip_entry(name): + """A deflated ZIP entry with a fixed date and attributes, so the same input gives the same bytes.""" + info = zipfile.ZipInfo(name, date_time=_PACKAGE_DATE_TIME) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 0 + return info + + +def normalized_package(package, context): + """Copy a written Office package into a new spool with fixed ZIP dates and attributes. + + Office writers stamp the current time into every entry, so without this the same files + merged twice would differ and a replayed workflow would attach a second copy. + """ + package.seek(0) + spool = new_output_spool() + try: + with zipfile.ZipFile(package, "r") as source, zipfile.ZipFile( + spool, "w", zipfile.ZIP_DEFLATED, allowZip64=True, + ) as target: + for entry in source.infolist(): + context.check_cancel() + info = fixed_zip_entry(entry.filename) + # A known size lets zipfile add ZIP64 fields only to entries that need them. + info.file_size = entry.file_size + with source.open(entry) as reader, target.open(info, "w") as writer: + while chunk := reader.read(_COPY_CHUNK_BYTES): + writer.write(chunk) + except BaseException: + spool.close() + raise + return spool + + +def finish_output(context, spool): + """Check the written size and return the merge result.""" + spool.flush() + size = spool.seek(0, io.SEEK_END) + if size > context.limits.max_output_bytes: + spool.close() + raise DocumentMergeError( + "output_too_large", + f"The merged file would be larger than {context.limits.max_output_bytes // _MIB} MB.", + ) + spool.seek(0) + context.report["status"] = "merged" + context.report["totals"]["bytes"] = size + return DocumentMergeResult(context.kind, spool, size, context.report) + + +def selected_indices(part, total, unit): + """0-based indices for a part's 1-based inclusive ranges; every page or slide when None.""" + if not part.pages: + return list(range(total)) + if len(part.pages) > MAX_PAGE_RANGES: + raise DocumentMergeError("range_invalid", f"Too many {unit} ranges were requested for {part.display_name()}.") + indices = [] + for item in part.pages: + if ( + not isinstance(item, (tuple, list)) or len(item) != 2 + or any(type(value) is not int for value in item) + or item[0] < 1 or item[1] < item[0] or item[1] > total + ): + raise DocumentMergeError( + "range_invalid", + f"A requested {unit} range is outside {part.display_name()}, which has {total} {unit}s.", + ) + indices.extend(range(item[0] - 1, item[1])) + if len(set(indices)) != len(indices): + raise DocumentMergeError("range_invalid", f"Each {unit} of {part.display_name()} can be included only once.") + return indices + + +def guard_ooxml_package(content, part, limits, label, *, check=None): + """Refuse encrypted, damaged, oversized or unsafe Office Open XML packages before parsing. + + ``check()`` runs between parts; its errors, such as a cancellation, pass through unchanged. + """ + if content[:8] == _OLE_MAGIC: + raise DocumentMergeError( + "encrypted_document", + f"{part.display_name()} is password-protected or encrypted, so it can't be merged.", + ) + if content[:4] != _ZIP_MAGIC: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid {label}.") + try: + archive = zipfile.ZipFile(io.BytesIO(content)) + except zipfile.BadZipFile as exc: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid {label}.") from exc + with archive: + entries = archive.infolist() + if len(entries) > limits.max_package_entries: + raise DocumentMergeError("source_too_large", f"{part.display_name()} has too many internal parts to merge.") + if sum(entry.file_size for entry in entries) > limits.max_package_uncompressed_bytes: + raise DocumentMergeError("source_too_large", f"{part.display_name()} is too large to merge once uncompressed.") + try: + unsafe = first_unsafe_xml_part(archive, entries, check=check) + except UnreadablePackageError as exc: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid {label}.") from exc + if unsafe is not None: + raise DocumentMergeError( + "unreadable_document", + f"{part.display_name()} isn't a valid {label}; it has XML that Office files can't contain.", + ) + names = {entry.filename for entry in entries} + return names + + + +__all__ = [ + "DOCUMENT_MERGE_KINDS", + "DOCUMENT_MERGE_OUTPUTS", + "DOCUMENT_MERGE_REPORT_VERSION", + "DOCUMENT_MERGE_SOURCE_EXTENSIONS", + "DOCUMENT_MERGE_FORMATTING", + "DocumentMergeCancelled", + "DocumentMergeError", + "DocumentMergeLimits", + "DocumentMergeOptions", + "DocumentMergePart", + "DocumentMergeResult", + "FORMATTING_KEEP_SOURCE", + "FORMATTING_USE_FIRST", + "MERGE_KIND_DOCX", + "MERGE_KIND_PDF", + "MERGE_KIND_PPTX", + "MERGE_KIND_WORKBOOK", + "MergeContext", + "PART_STATUS_MERGED", + "PART_STATUS_NOT_PROCESSED", + "WORKBOOK_SHEETS_ALL", + "WORKBOOK_SHEETS_FIRST", + "WORKBOOK_SHEET_MODES", + "finish_output", + "fixed_zip_entry", + "guard_ooxml_package", + "new_output_spool", + "normalized_package", + "selected_indices", +] diff --git a/application/single_app/functions_document_merge_docx.py b/application/single_app/functions_document_merge_docx.py new file mode 100644 index 000000000..724fdb5d9 --- /dev/null +++ b/application/single_app/functions_document_merge_docx.py @@ -0,0 +1,312 @@ +# functions_document_merge_docx.py +"""Word assembly for V2 file merge: documents appended in order into one .docx. + +Version: 0.261.248 + +Composition uses docxcompose, which carries over each document's styles, numbering, +images, footnotes, diagrams and shapes. With ``keep_source`` formatting, a style that +shares a name with the first document's but looks different is copied under a new name, +so every document keeps its own look; with ``use_first``, the first document's styles +win. The merged file uses the first document's page setup, headers and footers, and a +fixed modified date, and the IDs of copied lists are derived rather than random, so the +same files always give the same bytes. Comments are not carried over. A document with +fields that start other programs (DDE) is refused, and the first document's link to its +template is removed so Word never fetches it; other fields and links are copied as they +are, and linked content is reported. +""" + +from datetime import datetime +import hashlib +import io +import re + +from functions_document_merge_core import ( + DocumentMergeError, + FORMATTING_KEEP_SOURCE, + finish_output, + guard_ooxml_package, + new_output_spool, + normalized_package, +) + + +_W_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +_R_NAMESPACE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +_W = f"{{{_W_NAMESPACE}}}" +_COMMENT_TAGS = ("commentRangeStart", "commentRangeEnd", "commentReference") +_PACKAGE_DATE = datetime(2000, 1, 1) +# DDE and DDEAUTO fields ask Word to start another program and read data from it. +_PROGRAM_FIELD = re.compile(r"\s*DDE(?:AUTO)?(?!\w)", re.IGNORECASE) +_FIELD_TAGS = (f"{_W}fldSimple", f"{_W}fldChar", f"{_W}instrText", f"{_W}delInstrText") +_RT_SETTINGS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/settings" +_RT_HYPERLINK = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/hyperlink" +_RT_NUMBERING = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" +_LIST_ID_SEED = b"simplechat/file-merge/word-list-id:" + + +def assemble_docx(context): + """Append every part's body, in order, to the first document.""" + # python-docx and docxcompose load only when a Word merge runs. + from docx import Document + from docxcompose.composer import Composer + + composer = None + master = None + first_lists = set() + for index, part in enumerate(context.parts): + content = context.load(index) + guard_ooxml_package(content, part, context.limits, "Word document", check=context.check_cancel) + try: + document = Document(io.BytesIO(content)) + starts_programs = _has_program_fields(document) + except Exception as exc: + if not context.blames_file(exc): + raise + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} couldn't be opened as a Word document.", + ) from exc + if starts_programs: + raise DocumentMergeError( + "active_content", + f"{part.display_name()} has fields that start other programs (DDE), so it can't be merged.", + ) + entry = context.part_entry(index) + entry["paragraphs"] = len(document.paragraphs) + entry["tables"] = len(document.tables) + context.check_cancel() + try: + if index == 0: + master = document + first_lists = _list_definition_ids(master) + if _remove_linked_template(master): + context.warn( + "linked_template_removed", + "This document's link to its template was removed, so Word won't fetch the template.", + index, + ) + composer = Composer(master, preserve_styles=context.options.formatting == FORMATTING_KEEP_SOURCE) + if context.options.source_headings: + _insert_heading(master, part.display_name(), at_start=True) + else: + if _remove_comment_marks(document): + context.warn("comments_removed", "Comments in this document were not carried over.", index) + if context.options.page_breaks: + master.add_page_break() + if context.options.source_headings: + _insert_heading(master, part.display_name(), at_start=False) + composer.append(document) + except DocumentMergeError: + raise + except Exception as exc: + if not context.blames_file(exc): + raise + # Library errors can quote document text, so only the file's name is reported. + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} couldn't be combined with the other documents.", + ) from exc + context.progress(index) + + try: + _derive_list_ids(master, first_lists) + linked = _linked_content_count(master) + except Exception as exc: + if not context.blames_file(exc): + raise + raise DocumentMergeError("merge_failed", "The merged Word document couldn't be prepared.") from exc + if linked: + context.warn( + "linked_content", + "The merged document links to content in other files or on the web; the links were kept.", + ) + context.limit("The merged document uses the first document's page setup, headers and footers.") + if context.options.formatting == FORMATTING_KEEP_SOURCE: + context.limit( + "Styles that share a name but look different are kept as separate copies, so each document keeps its look." + ) + else: + context.limit("Styles that share a name use the first document's definition.") + return finish_output(context, _write_package(composer, master, context)) + + +def _write_package(composer, master, context): + """Save the composed document with a fixed modified date and ZIP metadata, then re-open it.""" + from docx import Document + + package = new_output_spool() + spool = None + try: + # A document without core properties gets defaults stamped with the current time. + master.core_properties.modified = _PACKAGE_DATE + composer.save(package) + spool = normalized_package(package, context) + # Re-open the result so a malformed package fails here, not in the user's Word. + Document(spool) + return spool + except Exception as exc: + if spool is not None: + spool.close() + if isinstance(exc, DocumentMergeError) or not context.blames_file(exc): + raise + raise DocumentMergeError("merge_failed", "The merged Word document couldn't be written.") from exc + except BaseException: + if spool is not None: + spool.close() + raise + finally: + package.close() + + +def _insert_heading(document, text, *, at_start): + """Add a heading naming a source document, using Heading 1 when the document has it.""" + try: + paragraph = document.add_heading(text, level=1) + except KeyError: + paragraph = document.add_paragraph() + paragraph.add_run(text).bold = True + if at_start: + element = paragraph._p + body = element.getparent() + body.remove(element) + body.insert(0, element) + return paragraph + + +def _remove_comment_marks(document): + """Comments live in a part that is not carried over, so remove their anchors.""" + removed = False + for tag in _COMMENT_TAGS: + for element in document.element.body.iter(f"{{{_W_NAMESPACE}}}{tag}"): + removed = True + if not removed: + return False + for tag in _COMMENT_TAGS: + for element in list(document.element.body.iter(f"{{{_W_NAMESPACE}}}{tag}")): + parent = element.getparent() + if tag == "commentReference" and parent is not None and parent.tag == f"{{{_W_NAMESPACE}}}r": + run_parent = parent.getparent() + if run_parent is not None: + run_parent.remove(parent) + continue + if parent is not None: + parent.remove(element) + return True + + +def _part_root(part): + """A Word XML part's tree; a part python-docx keeps as bytes is parsed with entities off.""" + element = getattr(part, "_element", None) + if element is not None: + return element + content_type = str(getattr(part, "content_type", "") or "") + if "wordprocessingml" not in content_type or not content_type.endswith("+xml"): + return None + from lxml import etree + + parser = etree.XMLParser(resolve_entities=False, no_network=True, huge_tree=False) + return etree.fromstring(part.blob, parser) + + +def _has_program_fields(document): + """Whether a field anywhere in the document, its headers, footers or notes starts a program.""" + for part in document.part.package.iter_parts(): + root = _part_root(part) + if root is not None and _starts_programs(root): + return True + return False + + +def _starts_programs(root): + # A complex field's code sits between its "begin" and "separate" (or "end") marks; it can be + # split across runs and contain other fields, so the code of every open field is collected. + codes = [] + for element in root.iter(*_FIELD_TAGS): + if element.tag == f"{_W}fldSimple": + if _PROGRAM_FIELD.match(element.get(f"{_W}instr") or ""): + return True + elif element.tag == f"{_W}fldChar": + kind = element.get(f"{_W}fldCharType") + if kind == "begin": + codes.append([]) + elif kind in ("separate", "end") and codes: + if codes[-1] is not None and _PROGRAM_FIELD.match("".join(codes[-1])): + return True + codes[-1] = None + if kind == "end": + codes.pop() + elif codes and codes[-1] is not None: + codes[-1].append(element.text or "") + # A field still open at the end of a part counts too. + return any(code is not None and _PROGRAM_FIELD.match("".join(code)) for code in codes) + + +def _remove_linked_template(document): + """Remove the link to the document's template, so Word never fetches it; True when removed.""" + settings = next( + (rel.target_part for rel in document.part.rels.values() if rel.reltype == _RT_SETTINGS and not rel.is_external), + None, + ) + root = getattr(settings, "_element", None) + if root is None: + return False + templates = list(root.iter(f"{_W}attachedTemplate")) + for template in templates: + rid = template.get(f"{{{_R_NAMESPACE}}}id") + template.getparent().remove(template) + if rid and rid in settings.rels: + settings.drop_rel(rid) + return bool(templates) + + +def _linked_content_count(document): + """How many relationships in the merged package point at other files or the web, links aside.""" + return sum( + 1 for part in document.part.package.iter_parts() for rel in part.rels.values() + if rel.is_external and rel.reltype != _RT_HYPERLINK + ) + + +def _numbering_root(document): + """The document's numbering XML, without creating a numbering part; None when there is none.""" + for rel in document.part.rels.values(): + if rel.reltype == _RT_NUMBERING and not rel.is_external: + return getattr(rel.target_part, "_element", None) + return None + + +def _list_definition_ids(document): + root = _numbering_root(document) + if root is None: + return set() + return {element.get(f"{_W}abstractNumId") for element in root.iter(f"{_W}abstractNum")} + + +def _derive_list_ids(document, first_lists): + """Give each list definition docxcompose copied a derived ID instead of its random one. + + docxcompose gives every copied list a new random ``w:nsid``, so each later document's + lists restart rather than continue the first's. A value derived from the definition's + number, unique within the document, does the same and keeps the bytes reproducible. + """ + root = _numbering_root(document) + if root is None: + return + definitions = list(root.iter(f"{_W}abstractNum")) + copied = [element for element in definitions if element.get(f"{_W}abstractNumId") not in first_lists] + taken = { + (nsid.get(f"{_W}val") or "").upper() + for element in definitions if element.get(f"{_W}abstractNumId") in first_lists + for nsid in element.iter(f"{_W}nsid") + } + for element in copied: + nsid = next(element.iter(f"{_W}nsid"), None) + if nsid is None: + continue + seed = hashlib.sha256(_LIST_ID_SEED + (element.get(f"{_W}abstractNumId") or "").encode("utf-8")).digest() + value = int.from_bytes(seed[:4], "big") + while f"{value:08X}" in taken: + value = (value + 1) & 0xFFFFFFFF + nsid.set(f"{_W}val", f"{value:08X}") + taken.add(f"{value:08X}") + + +__all__ = ["assemble_docx"] diff --git a/application/single_app/functions_document_merge_pdf.py b/application/single_app/functions_document_merge_pdf.py new file mode 100644 index 000000000..309623646 --- /dev/null +++ b/application/single_app/functions_document_merge_pdf.py @@ -0,0 +1,255 @@ +# functions_document_merge_pdf.py +"""PDF assembly for V2 file merge: ordered pages, one bookmark per source file. + +Version: 0.261.248 + +Pages are copied with pypdf, so text, images, links and fonts stay exactly as they were. +Each source becomes a top-level bookmark that keeps the source's own bookmarks beneath +it. Encrypted PDFs are refused rather than decrypted, so a merge never removes a +document's protection. Actions that could run code, open files or programs, or submit +forms are removed wherever they appear: open actions, document, page, link and +form-field scripts, and XFA form logic. Only navigation is kept: links to a page, the +standard next, previous, first and last page commands, and http, https and mailto +addresses. + +pypdf keeps every source in memory until the merged file is written, so memory grows with +the total input and callers bound it with ``max_total_input_bytes``. This module, and so +pypdf, loads only when a PDF merge runs. +""" + +import io +import re + +from pypdf import PdfReader, PdfWriter +from pypdf.generic import ArrayObject, DictionaryObject, NullObject + +from functions_document_merge_core import ( + DocumentMergeError, + MERGE_KIND_PDF, + finish_output, + new_output_spool, + selected_indices, +) + + +_PDF_HEADER = b"%PDF-" +_HEADER_SEARCH_BYTES = 1024 +_MAX_ACTION_CHAIN = 64 +_STANDARD_NAMED_ACTIONS = frozenset({"/NextPage", "/PrevPage", "/FirstPage", "/LastPage"}) +_WEB_AND_EMAIL_SCHEMES = frozenset({"http", "https", "mailto"}) +# URL parsers drop control characters and spaces before they read a scheme. +_URI_IGNORED_CHARACTERS = re.compile(r"[\x00-\x20\x7f]") + + +def assemble_pdf(context): + """Append every part's selected pages, in order, into one PDF.""" + writer = PdfWriter() + total_pages = 0 + for index, part in enumerate(context.parts): + copied, source_pages, has_form_fields = _append_part(writer, context, index, part, total_pages) + entry = context.part_entry(index) + entry["pages"] = copied + entry["source_pages"] = source_pages + total_pages += copied + if has_form_fields: + context.warn( + "form_fields", "This PDF has fillable form fields; fields with the same name share one value.", + index, + ) + context.progress(index) + + try: + removed = _strip_active_content(writer) + except Exception as exc: + if not context.blames_file(exc): + raise + # A file that can't be checked for scripts is not delivered. + raise DocumentMergeError("merge_failed", "The merged PDF couldn't be checked for scripts.") from exc + if removed: + context.warn( + "active_content_removed", + "Scripts, form actions and links that open files or programs were removed; links to pages and web " + "or email addresses were kept.", + ) + if context.options.bookmarks: + context.limit("Each file starts with a bookmark named after it; its own bookmarks are kept beneath it.") + context.report["totals"]["pages"] = total_pages + context.check_cancel() + spool = new_output_spool() + try: + writer.write(spool) + except Exception as exc: + spool.close() + if not context.blames_file(exc): + raise + raise DocumentMergeError("merge_failed", "The merged PDF couldn't be written.") from exc + except BaseException: + spool.close() + raise + return finish_output(context, spool) + + +def _append_part(writer, context, index, part, pages_so_far): + """Copy one source's selected pages into the writer.""" + content = context.load(index) + if _PDF_HEADER not in content[:_HEADER_SEARCH_BYTES]: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid PDF.") + try: + reader = PdfReader(io.BytesIO(content), strict=False) + if reader.is_encrypted: + raise DocumentMergeError( + "encrypted_document", + f"{part.display_name()} is password-protected or encrypted, so it can't be merged.", + ) + page_count = len(reader.pages) + except DocumentMergeError: + raise + except Exception as exc: + if not context.blames_file(exc): + raise + raise DocumentMergeError("unreadable_document", f"{part.display_name()} couldn't be read as a PDF.") from exc + if page_count == 0: + raise DocumentMergeError("empty_source", f"{part.display_name()} has no pages.") + indices = selected_indices(part, page_count, "page") + if pages_so_far + len(indices) > context.limits.max_pages: + raise DocumentMergeError( + "page_limit_exceeded", f"The merged PDF would have more than {context.limits.max_pages:,} pages.", + ) + context.check_cancel() + try: + writer.append( + reader, + outline_item=part.display_name() if context.options.bookmarks else None, + pages=indices, + import_outline=context.options.bookmarks, + ) + has_form_fields = _has_form_fields(reader) + except Exception as exc: + if not context.blames_file(exc): + raise + raise DocumentMergeError("unreadable_document", f"{part.display_name()} couldn't be read as a PDF.") from exc + finally: + # pypdf keys its copy table by id(reader); dropping it means a later source can't reuse it. + writer.reset_translation(reader) + return len(indices), page_count, has_form_fields + + +def _has_form_fields(reader): + try: + root = reader.trailer["/Root"] + return "/AcroForm" in root + except (KeyError, TypeError): + return False + + +def _strip_active_content(writer): + """Remove actions that could run code, open programs or submit forms; return how many.""" + root = writer.root_object + removed = _drop_keys(root, ("/OpenAction", "/AA")) + names = _resolved(root.get("/Names")) + if isinstance(names, DictionaryObject): + removed += _drop_keys(names, ("/JavaScript",)) + for page in writer.pages: + removed += _strip_actions(page) + annotations = _resolved(page.get("/Annots")) + if isinstance(annotations, ArrayObject): + for annotation in annotations: + annotation = _resolved(annotation) + if isinstance(annotation, DictionaryObject): + removed += _strip_actions(annotation) + form = _resolved(root.get("/AcroForm")) + if isinstance(form, DictionaryObject): + removed += _drop_keys(form, ("/XFA",)) + removed += _strip_tree(form.get("/Fields"), ("/Kids",)) + outlines = _resolved(root.get("/Outlines")) + if isinstance(outlines, DictionaryObject): + removed += _strip_tree(outlines.get("/First"), ("/First", "/Next")) + return removed + + +def _strip_tree(start, child_keys): + """Strip actions from every node reachable through child_keys, visiting each node once.""" + removed = 0 + pending = [start] + seen = set() + while pending: + node = _resolved(pending.pop()) + if not isinstance(node, (ArrayObject, DictionaryObject)) or id(node) in seen: + continue + seen.add(id(node)) + if isinstance(node, ArrayObject): + pending.extend(node) + continue + removed += _strip_actions(node) + pending.extend(node.get(key) for key in child_keys if key in node) + return removed + + +def _strip_actions(item): + removed = _drop_keys(item, ("/AA",)) + if "/A" in item and not _is_safe_action(item.get("/A")): + del item["/A"] + removed += 1 + return removed + + +def _is_safe_action(action): + """True when an action, and every action chained after it, only navigates.""" + pending = [action] + visited = 0 + while pending: + current = _resolved(pending.pop()) + visited += 1 + if visited > _MAX_ACTION_CHAIN or not isinstance(current, DictionaryObject) or not _navigates(current): + return False + following = _resolved(current.get("/Next")) + if isinstance(following, ArrayObject): + pending.extend(following) + elif following is not None: + pending.append(following) + return True + + +def _navigates(action): + """Only a page in this file, a standard page command, or a web or email address.""" + action_type = _resolved(action.get("/S")) + if action_type == "/GoTo": + return True + if action_type == "/Named": + command = _resolved(action.get("/N")) + return isinstance(command, str) and command in _STANDARD_NAMED_ACTIONS + if action_type == "/URI": + return _is_web_or_email(_resolved(action.get("/URI"))) + return False + + +def _is_web_or_email(value): + if isinstance(value, bytes): + value = value.decode("latin-1", "replace") + if not isinstance(value, str): + return False + text = "".join(_URI_IGNORED_CHARACTERS.sub("", value).split()) + scheme, separator, _rest = text.partition(":") + return bool(separator) and scheme.lower() in _WEB_AND_EMAIL_SCHEMES + + +def _drop_keys(item, keys): + removed = 0 + for key in keys: + if key in item: + del item[key] + removed += 1 + return removed + + +def _resolved(value): + if value is None: + return None + try: + value = value.get_object() + except Exception: + return None + return None if value is None or isinstance(value, NullObject) else value + + +__all__ = ["MERGE_KIND_PDF", "assemble_pdf"] diff --git a/application/single_app/functions_document_merge_pptx.py b/application/single_app/functions_document_merge_pptx.py new file mode 100644 index 000000000..4ef31af47 --- /dev/null +++ b/application/single_app/functions_document_merge_pptx.py @@ -0,0 +1,792 @@ +# functions_document_merge_pptx.py +"""PowerPoint assembly for V2 file merge: slides appended in order into one deck. + +Version: 0.261.248 + +The merge works on the Office Open XML package itself, so every slide is copied with +everything it relates to — pictures, media, charts and their embedded workbooks, +diagrams and speaker notes — exactly as it was. ``keep_source`` formatting brings each +deck's slide layouts, masters and themes along (identical ones are reused), so slides +keep their original look. ``use_first`` places each slide on the first deck's matching +layout, by name and then by type, so every slide takes the first deck's theme. Each +source deck can become its own section. The first deck decides the slide size. Entries +and section IDs are fixed, so the same decks always give the same bytes. Click and hover +actions that start a program or run a macro are removed from every deck; hyperlinks are +copied as they are, and content linked from other files or the web stays linked and is +reported. +""" + +import hashlib +import io +import posixpath +import re +import uuid +import zipfile + +from functions_document_merge_core import ( + DocumentMergeError, + FORMATTING_KEEP_SOURCE, + finish_output, + fixed_zip_entry, + guard_ooxml_package, + new_output_spool, + selected_indices, +) +from functions_ooxml_package_guard import content_type_of, is_office_xml_part + + +_NS = { + "p": "http://schemas.openxmlformats.org/presentationml/2006/main", + "a": "http://schemas.openxmlformats.org/drawingml/2006/main", + "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "rel": "http://schemas.openxmlformats.org/package/2006/relationships", + "ct": "http://schemas.openxmlformats.org/package/2006/content-types", + "p14": "http://schemas.microsoft.com/office/powerpoint/2010/main", +} +_RT = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" +RT_OFFICE_DOCUMENT = _RT + "officeDocument" +RT_SLIDE = _RT + "slide" +RT_SLIDE_LAYOUT = _RT + "slideLayout" +RT_SLIDE_MASTER = _RT + "slideMaster" +RT_NOTES_MASTER = _RT + "notesMaster" +RT_HYPERLINK = _RT + "hyperlink" +_COMMENT_RELATIONSHIP_SUFFIXES = ("/comments", "/commentAuthors") +_SECTION_EXTENSION_URI = "{521415D9-36F7-43E2-AB2F-B90AF26B5E84}" +_SECTION_ID_NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/microsoft/simplechat/file-merge/sections") +_FIRST_MASTER_OR_LAYOUT_ID = 2147483648 +_FIRST_SLIDE_ID = 256 +_MAX_SLIDE_ID = 2147483647 +_PRESENTATION_CHILD_ORDER = ( + "sldMasterIdLst", "notesMasterIdLst", "handoutMasterIdLst", "sldIdLst", "sldSz", "notesSz", +) +_NUMBERED_NAME = re.compile(r"^(?P.*?)(?P\d*)(?P\.[^./]+)$") +# Slide actions that start another program or run a macro instead of following a link. +_PROGRAM_ACTIONS = ("ppaction://program", "ppaction://macro") +_ACTION_TAGS = ("hlinkClick", "hlinkHover", "hlinkMouseOver") +_RELATIONSHIPS_TYPE = "application/vnd.openxmlformats-package.relationships+xml" + + +def _parser(): + # lxml ships with python-pptx; entity resolution and network access stay off. + from lxml import etree + + return etree.XMLParser(resolve_entities=False, no_network=True, huge_tree=False, remove_blank_text=False) + + +def _parse(data): + from lxml import etree + + return etree.fromstring(data, _parser()) + + +def _serialize(element): + from lxml import etree + + return etree.tostring(element, xml_declaration=True, encoding="UTF-8", standalone=True) + + +def _q(prefix, name): + return f"{{{_NS[prefix]}}}{name}" + + +class _Relationship: + __slots__ = ("rid", "reltype", "target", "external") + + def __init__(self, rid, reltype, target, external): + self.rid = rid + self.reltype = reltype + self.target = target + self.external = external + + +class _Package: + """An Office Open XML package as part names, bytes, relationships and content types.""" + + def __init__(self, content, part, limits, check=None): + guard_ooxml_package(content, part, limits, "PowerPoint presentation", check=check) + try: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + self.files = {info.filename: archive.read(info) for info in archive.infolist() if not info.is_dir()} + except (zipfile.BadZipFile, KeyError, RuntimeError) as exc: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid PowerPoint presentation.") from exc + self.part = part + if "[Content_Types].xml" not in self.files: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid PowerPoint presentation.") + types = _parse(self.files["[Content_Types].xml"]) + self.defaults = { + element.get("Extension", "").lower(): element.get("ContentType") + for element in types.iter(_q("ct", "Default")) + } + self.overrides = { + element.get("PartName", "").lstrip("/"): element.get("ContentType") + for element in types.iter(_q("ct", "Override")) + } + root_rels = self.relationships("") + documents = [rel for rel in root_rels if rel.reltype == RT_OFFICE_DOCUMENT and not rel.external] + if not documents: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} isn't a valid PowerPoint presentation.") + self.presentation_name = documents[0].target + content_type = self.content_type(self.presentation_name) or "" + if "presentationml" not in content_type or "macroEnabled" in content_type: + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} isn't a PowerPoint (.pptx) presentation.", + ) + + @staticmethod + def rels_name(part_name): + directory, file_name = posixpath.split(part_name) + return posixpath.join(directory, "_rels", f"{file_name}.rels") + + def relationships(self, part_name): + data = self.files.get(self.rels_name(part_name) if part_name else "_rels/.rels") + if data is None: + return [] + directory = posixpath.dirname(part_name) + rels = [] + for element in _parse(data).iter(_q("rel", "Relationship")): + external = element.get("TargetMode") == "External" + target = element.get("Target", "") + if not external: + target = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join(directory, target), + ) + rels.append(_Relationship(element.get("Id"), element.get("Type"), target, external)) + return rels + + def content_type(self, part_name): + if part_name in self.overrides: + return self.overrides[part_name] + extension = posixpath.splitext(part_name)[1].lstrip(".").lower() + return self.defaults.get(extension) + + def slide_names(self): + presentation = _parse(self.files[self.presentation_name]) + by_id = {rel.rid: rel.target for rel in self.relationships(self.presentation_name)} + names = [] + for element in presentation.iter(_q("p", "sldId")): + target = by_id.get(element.get(_q("r", "id"))) + if target in self.files: + names.append(target) + return names + + def slide_size(self): + size = _parse(self.files[self.presentation_name]).find(_q("p", "sldSz")) + return (size.get("cx"), size.get("cy")) if size is not None else None + + +def _write_relationships(rels, owner=None): + """Serialize relationships; with ``owner``, internal targets are absolute names made relative.""" + from lxml import etree + + root = etree.Element(_q("rel", "Relationships"), nsmap={None: _NS["rel"]}) + for rel in rels: + target = rel.target if rel.external or owner is None else _relative(owner, rel.target) + attributes = {"Id": rel.rid, "Type": rel.reltype, "Target": target} + if rel.external: + attributes["TargetMode"] = "External" + etree.SubElement(root, _q("rel", "Relationship"), attributes) + return _serialize(root) + + +def _relative(from_part, to_part): + return posixpath.relpath(to_part, posixpath.dirname(from_part) or ".") + + +class _Destination: + """The first deck, growing as other decks' slides and parts are copied in.""" + + def __init__(self, package, context): + self.package = package + self.context = context + self.files = dict(package.files) + self.defaults = dict(package.defaults) + self.overrides = dict(package.overrides) + self.used = {name.lower() for name in self.files} + self.presentation_name = package.presentation_name + self.presentation = _parse(self.files[self.presentation_name]) + self.presentation_rels = package.relationships(self.presentation_name) + self.layout_fingerprints = {} + self.layouts = [] + self.notes_master = next( + (rel.target for rel in self.presentation_rels if rel.reltype == RT_NOTES_MASTER and not rel.external), None, + ) + self._index_layouts(package) + + def _index_layouts(self, package): + for rel in self.presentation_rels: + if rel.reltype != RT_SLIDE_MASTER or rel.external: + continue + master_name = rel.target + master_print = _master_fingerprint(package, master_name) + for layout_rel in package.relationships(master_name): + if layout_rel.reltype != RT_SLIDE_LAYOUT or layout_rel.external: + continue + layout_name = layout_rel.target + self.layout_fingerprints.setdefault( + _layout_fingerprint(package, layout_name, master_print), layout_name, + ) + self.layouts.append((layout_name, _layout_identity(package.files.get(layout_name, b"")))) + + def unique_name(self, name): + if name.lower() not in self.used: + self.used.add(name.lower()) + return name + directory, file_name = posixpath.split(name) + match = _NUMBERED_NAME.match(file_name) + stem, extension = (match.group("stem"), match.group("extension")) if match else (file_name, "") + number = 1 + while True: + candidate = posixpath.join(directory, f"{stem}{number}{extension}") + if candidate.lower() not in self.used: + self.used.add(candidate.lower()) + return candidate + number += 1 + + def next_presentation_rid(self): + taken = {rel.rid for rel in self.presentation_rels} + number = len(taken) + 1 + while f"rId{number}" in taken: + number += 1 + return f"rId{number}" + + def add_presentation_rel(self, reltype, part_name): + rid = self.next_presentation_rid() + self.presentation_rels.append(_Relationship(rid, reltype, part_name, False)) + return rid + + def add_content_type(self, source, source_name, new_name): + if source_name in source.overrides: + self.overrides[new_name] = source.overrides[source_name] + return + extension = posixpath.splitext(new_name)[1].lstrip(".").lower() + if extension not in self.defaults: + content_type = source.content_type(source_name) + if content_type: + self.defaults[extension] = content_type + + def child(self, name): + element = self.presentation.find(_q("p", name)) + if element is not None: + return element + from lxml import etree + + element = etree.Element(_q("p", name)) + position = _PRESENTATION_CHILD_ORDER.index(name) + anchor = 0 + for index, existing in enumerate(self.presentation): + local = etree.QName(existing).localname + if local in _PRESENTATION_CHILD_ORDER and _PRESENTATION_CHILD_ORDER.index(local) < position: + anchor = index + 1 + self.presentation.insert(anchor, element) + return element + + def next_master_or_layout_id(self): + values = [_FIRST_MASTER_OR_LAYOUT_ID - 1] + values.extend(int(element.get("id")) for element in self.presentation.iter(_q("p", "sldMasterId"))) + # Masters are found through the presentation, whatever their names or encodings. + for rel in self.presentation_rels: + if rel.reltype == RT_SLIDE_MASTER and not rel.external and rel.target in self.files: + values.extend( + int(element.get("id")) for element in _parse(self.files[rel.target]).iter(_q("p", "sldLayoutId")) + ) + return max(values) + 1 + + def next_slide_id(self): + ids = [int(element.get("id")) for element in self.presentation.iter(_q("p", "sldId"))] + value = max(ids + [_FIRST_SLIDE_ID - 1]) + 1 + if value > _MAX_SLIDE_ID: + raise DocumentMergeError("slide_limit_exceeded", "The merged deck has too many slides.") + return value + + def write(self, spool): + from lxml import etree + + self.files[self.presentation_name] = _serialize(self.presentation) + self.files[_Package.rels_name(self.presentation_name)] = _write_relationships( + self.presentation_rels, owner=self.presentation_name, + ) + types = etree.Element(_q("ct", "Types"), nsmap={None: _NS["ct"]}) + for extension, content_type in sorted(self.defaults.items()): + etree.SubElement(types, _q("ct", "Default"), {"Extension": extension, "ContentType": content_type}) + for part_name, content_type in sorted(self.overrides.items()): + if part_name in self.files: + etree.SubElement(types, _q("ct", "Override"), {"PartName": f"/{part_name}", "ContentType": content_type}) + with zipfile.ZipFile(spool, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(fixed_zip_entry("[Content_Types].xml"), _serialize(types)) + for name in sorted(self.files): + if name != "[Content_Types].xml": + archive.writestr(fixed_zip_entry(name), self.files[name]) + + +def _master_fingerprint(package, master_name): + digest = hashlib.sha256(package.files.get(master_name, b"")) + for rel in package.relationships(master_name): + if not rel.external and rel.reltype.endswith("/theme"): + digest.update(package.files.get(rel.target, b"")) + return digest.hexdigest() + + +def _layout_fingerprint(package, layout_name, master_print): + return hashlib.sha256(package.files.get(layout_name, b"") + master_print.encode("ascii")).hexdigest() + + +def _layout_identity(data): + """A layout's name and type, used to match layouts across decks.""" + if not data: + return ("", "") + layout = _parse(data) + common = layout.find(_q("p", "cSld")) + return ( + ((common.get("name") if common is not None else "") or "").strip().casefold(), + (layout.get("type") or "").strip(), + ) + + +def _master_of_layout(package, layout_name): + return next( + (rel.target for rel in package.relationships(layout_name) if rel.reltype == RT_SLIDE_MASTER and not rel.external), + None, + ) + + +class _DeckCopier: + """Copies one source deck's selected slides into the destination.""" + + def __init__(self, destination, source, index, slide_names): + self.destination = destination + self.source = source + self.index = index + self.mapping = {} + self.copied_masters = [] + self.keep_source = destination.context.options.formatting == FORMATTING_KEEP_SOURCE + self.fallback_layouts = 0 + self.dropped_comments = 0 + self.dropped_links = 0 + self.linked_content = 0 + self.slide_names = set(slide_names) + self.source_masters = { + rel.target for rel in source.relationships(source.presentation_name) + if rel.reltype == RT_SLIDE_MASTER and not rel.external + } + for name in slide_names: + self.mapping[name] = destination.unique_name(name) + + def copy_slide(self, slide_name): + return self._copy(slide_name, force=True) + + def _copy(self, source_name, force=False): + if source_name in self.mapping and not force: + return self.mapping[source_name] + new_name = self.mapping.get(source_name) or self.destination.unique_name(source_name) + self.mapping[source_name] = new_name + data = self.source.files[source_name] + new_rels = [] + dropped = set() + for rel in self.source.relationships(source_name): + if rel.external: + if rel.reltype != RT_HYPERLINK: + self.linked_content += 1 + new_rels.append(rel) + continue + target = self._target_for(source_name, rel) + if target is None: + dropped.add(rel.rid) + continue + new_rels.append(_Relationship(rel.rid, rel.reltype, _relative(new_name, target), False)) + if dropped: + data = _remove_references(data, dropped) + if source_name in self.source_masters: + self.copied_masters.append(new_name) + self.destination.files[new_name] = data + if new_rels: + self.destination.files[_Package.rels_name(new_name)] = _write_relationships(new_rels) + self.destination.add_content_type(self.source, source_name, new_name) + return new_name + + def _target_for(self, source_name, rel): + """The destination part for a relationship, or None when it is not carried over.""" + target = rel.target + if target not in self.source.files: + return None + if rel.reltype.endswith(_COMMENT_RELATIONSHIP_SUFFIXES) or "/comments" in rel.reltype: + self.dropped_comments += 1 + return None + if rel.reltype == RT_SLIDE: + if target in self.slide_names or target in self.mapping: + return self.mapping.get(target) or self._copy(target) + self.dropped_links += 1 + return None + if rel.reltype == RT_SLIDE_LAYOUT and source_name in self.slide_names: + return self._layout_for(target) + if rel.reltype == RT_NOTES_MASTER: + if self.destination.notes_master is not None: + return self.destination.notes_master + copied = self._copy(target) + rid = self.destination.add_presentation_rel(RT_NOTES_MASTER, copied) + notes = self.destination.child("notesMasterIdLst") + from lxml import etree + + etree.SubElement(notes, _q("p", "notesMasterId"), {_q("r", "id"): rid}) + self.destination.notes_master = copied + return copied + return self._copy(target) + + def _layout_for(self, layout_name): + master_name = _master_of_layout(self.source, layout_name) + master_print = _master_fingerprint(self.source, master_name) if master_name else "" + fingerprint = _layout_fingerprint(self.source, layout_name, master_print) + existing = self.destination.layout_fingerprints.get(fingerprint) + if existing is not None: + return existing + if self.keep_source: + copied = self._copy(layout_name) + # Every layout of a copied master is now in the destination; reuse them by fingerprint. + for rel in self.source.relationships(master_name or ""): + if rel.reltype == RT_SLIDE_LAYOUT and rel.target in self.mapping: + self.destination.layout_fingerprints.setdefault( + _layout_fingerprint(self.source, rel.target, master_print), self.mapping[rel.target], + ) + return copied + name, kind = _layout_identity(self.source.files.get(layout_name, b"")) + for candidate, (candidate_name, candidate_kind) in self.destination.layouts: + if name and candidate_name == name: + return candidate + for candidate, (_, candidate_kind) in self.destination.layouts: + if kind and candidate_kind == kind: + return candidate + self.fallback_layouts += 1 + for candidate, (_, candidate_kind) in self.destination.layouts: + if candidate_kind == "blank": + return candidate + return self.destination.layouts[0][0] + + +def _remove_references(data, rids): + """Remove elements that point at relationships that were not carried over.""" + root = _parse(data) + attributes = (_q("r", "id"), _q("r", "embed"), _q("r", "link"), _q("r", "pict")) + for element in list(root.iter()): + if any(element.get(attribute) in rids for attribute in attributes): + parent = element.getparent() + if parent is not None: + parent.remove(element) + return _serialize(root) + + +class _IdAllocator: + """Hands out master and layout IDs that stay unique across the merged presentation.""" + + def __init__(self, destination): + self.next_value = destination.next_master_or_layout_id() + + def take(self): + value = self.next_value + self.next_value += 1 + return value + + +class _MergeState: + """What the decks merged so far have built: the destination package, IDs, sections and size.""" + + __slots__ = ("destination", "allocator", "sections", "first_size", "total_slides") + + def __init__(self): + self.destination = None + self.allocator = None + self.sections = [] + self.first_size = None + self.total_slides = 0 + + +def assemble_pptx(context): + """Append every part's selected slides, in order, to the first deck.""" + state = _MergeState() + for index, part in enumerate(context.parts): + content = context.load(index) + try: + _add_deck(context, state, index, part, content) + except DocumentMergeError: + raise + except Exception as exc: + if not context.blames_file(exc): + raise + # Library errors can quote slide text, so only the file's name is reported. + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} couldn't be combined with the other decks.", + ) from exc + context.progress(index) + + context.report["totals"]["slides"] = state.total_slides + if context.options.formatting == FORMATTING_KEEP_SOURCE: + context.limit("Slides keep their original layouts, masters and themes. The first deck decides the slide size.") + else: + context.limit("Slides use the first deck's theme and its matching layouts. The first deck decides the slide size.") + spool = new_output_spool() + try: + _collect_garbage(state.destination) + try: + stripped = _strip_program_actions(state.destination) + except Exception as exc: + if not context.blames_file(exc): + raise + # A deck that can't be checked for actions that start programs is not delivered. + raise DocumentMergeError( + "merge_failed", "The merged PowerPoint deck couldn't be checked for actions that start programs.", + ) from exc + if stripped: + context.warn( + "active_content_removed", + "Click and hover actions that start programs or run macros were removed; other links were kept.", + ) + if context.options.sections: + _write_sections(state.destination, state.sections) + state.destination.write(spool) + spool.seek(0) + _validate(spool, state.total_slides) + except DocumentMergeError: + spool.close() + raise + except Exception as exc: + spool.close() + if not context.blames_file(exc): + raise + raise DocumentMergeError("merge_failed", "The merged PowerPoint deck couldn't be written.") from exc + except BaseException: + spool.close() + raise + return finish_output(context, spool) + + +def _add_deck(context, state, index, part, content): + """Append one deck's selected slides; the first deck becomes the destination.""" + from lxml import etree + + package = _Package(content, part, context.limits, check=context.check_cancel) + slide_names = package.slide_names() + if not slide_names: + raise DocumentMergeError("empty_source", f"{part.display_name()} has no slides.") + chosen = [slide_names[position] for position in selected_indices(part, len(slide_names), "slide")] + if state.total_slides + len(chosen) > context.limits.max_slides: + raise DocumentMergeError( + "slide_limit_exceeded", f"The merged deck would have more than {context.limits.max_slides:,} slides.", + ) + context.check_cancel() + entry = context.part_entry(index) + entry["slides"] = len(chosen) + entry["source_slides"] = len(slide_names) + if index == 0: + state.destination = _Destination(package, context) + state.first_size = package.slide_size() + state.allocator = _IdAllocator(state.destination) + state.sections.append((part.display_name(), _keep_first_deck_slides(state.destination, chosen))) + state.total_slides += len(chosen) + return + destination = state.destination + if package.slide_size() != state.first_size: + context.warn( + "slide_size_differs", + "This deck's slide size differs from the first deck's, so its slides may look stretched or cropped.", + index, + ) + if _parse(package.files[package.presentation_name]).find(_q("p", "embeddedFontLst")) is not None: + context.warn("embedded_fonts", "Fonts embedded in this deck were not carried over.", index) + copier = _DeckCopier(destination, package, index, chosen) + slide_list = destination.child("sldIdLst") + section_ids = [] + for slide_name in chosen: + context.check_cancel() + new_name = copier.copy_slide(slide_name) + rid = destination.add_presentation_rel(RT_SLIDE, new_name) + slide_id = str(destination.next_slide_id()) + etree.SubElement(slide_list, _q("p", "sldId"), {"id": slide_id, _q("r", "id"): rid}) + section_ids.append(slide_id) + for master_name in copier.copied_masters: + master = _parse(destination.files[master_name]) + for layout_id in master.iter(_q("p", "sldLayoutId")): + layout_id.set("id", str(state.allocator.take())) + destination.files[master_name] = _serialize(master) + rid = destination.add_presentation_rel(RT_SLIDE_MASTER, master_name) + etree.SubElement(destination.child("sldMasterIdLst"), _q("p", "sldMasterId"), { + "id": str(state.allocator.take()), _q("r", "id"): rid, + }) + if copier.fallback_layouts: + context.warn( + "layout_fallback", + f"{copier.fallback_layouts} slide(s) had no matching layout in the first deck and use a blank layout.", + index, + ) + if copier.dropped_comments: + context.warn("comments_removed", "Slide comments were not carried over.", index) + if copier.dropped_links: + context.warn("links_removed", "Links to slides that were not merged were removed.", index) + if copier.linked_content: + context.warn( + "linked_content", "This deck links to content in other files or on the web; the links were kept.", + index, + ) + state.sections.append((part.display_name(), section_ids)) + state.total_slides += len(chosen) + + +def _keep_first_deck_slides(destination, chosen): + """Keep the first deck's selected slides, in the selected order; return their slide IDs.""" + from lxml import etree + + if len(set(chosen)) != len(chosen): + raise DocumentMergeError("range_invalid", "A slide can appear only once in the merged deck.") + slide_list = destination.child("sldIdLst") + targets = {rel.rid: rel.target for rel in destination.presentation_rels} + by_name = {} + for element in list(slide_list): + by_name[targets.get(element.get(_q("r", "id")))] = element + slide_list.remove(element) + kept_ids = [] + for name in chosen: + element = by_name.pop(name) + slide_list.append(element) + kept_ids.append(element.get("id")) + removed = {element.get(_q("r", "id")) for element in by_name.values()} + if removed: + destination.presentation_rels = [rel for rel in destination.presentation_rels if rel.rid not in removed] + return kept_ids + + +def _reachable_parts(destination): + """Every part reachable from the package root, using the presentation's current relationships.""" + seen = set() + stack = [rel.target for rel in destination.package.relationships("") if not rel.external] + while stack: + name = stack.pop() + if name in seen: + continue + seen.add(name) + if name == destination.presentation_name: + stack.extend(rel.target for rel in destination.presentation_rels if not rel.external) + continue + rels_data = destination.files.get(_Package.rels_name(name)) + if rels_data is None: + continue + directory = posixpath.dirname(name) + for element in _parse(rels_data).iter(_q("rel", "Relationship")): + if element.get("TargetMode") == "External": + continue + target = element.get("Target", "") + stack.append( + target.lstrip("/") if target.startswith("/") + else posixpath.normpath(posixpath.join(directory, target)) + ) + return seen + + +def _is_relationships_part(name): + """Relationship parts are the .rels files inside a _rels folder; other names are ordinary parts.""" + return name.endswith(".rels") and posixpath.basename(posixpath.dirname(name)) == "_rels" + + +def _collect_garbage(destination): + """Remove parts nothing refers to any more, such as the first deck's unselected slides.""" + reachable = _reachable_parts(destination) + for name in list(destination.files): + if name == "[Content_Types].xml" or _is_relationships_part(name): + continue + if name not in reachable: + del destination.files[name] + for name in list(destination.files): + if not _is_relationships_part(name) or name == "_rels/.rels": + continue + directory, file_name = posixpath.split(name) + owner = posixpath.join(posixpath.dirname(directory), file_name[: -len(".rels")]) + if owner not in destination.files and owner != destination.presentation_name: + del destination.files[name] + + +def _strip_program_actions(destination): + """Remove click and hover actions that start a program or run a macro; return how many. + + Every part PowerPoint reads as Office XML is checked, by its declared content type and + whatever its name or encoding. Relationship parts and legacy VML drawings, also known by + their declared type, can't carry these actions. + """ + tags = tuple(_q("a", tag) for tag in _ACTION_TAGS) + relationship_prefix = "{" + _NS["r"] + "}" + overrides = {name.lower(): content_type for name, content_type in destination.overrides.items()} + removed = 0 + for name in sorted(destination.files): + lowered = name.lower() + content_type = content_type_of(name, destination.defaults, overrides).strip().lower() + # PowerPoint reads a part by its declared type, so only the type rules a part out, never its name. + if ( + lowered == "[content_types].xml" or content_type == _RELATIONSHIPS_TYPE + or "vmldrawing" in content_type or (not content_type and not lowered.endswith(".xml")) + or not is_office_xml_part(name, content_type) + ): + continue + root = _parse(destination.files[name]) + dropped = set() + for element in list(root.iter(*tags)): + if (element.get("action") or "").strip().lower().startswith(_PROGRAM_ACTIONS): + dropped.add(element.get(_q("r", "id"))) + element.getparent().remove(element) + removed += 1 + if not dropped: + continue + destination.files[name] = _serialize(root) + still_used = { + value for element in root.iter() if isinstance(element.tag, str) + for key, value in element.attrib.items() if key.startswith(relationship_prefix) + } + unused = {rid for rid in dropped if rid and rid not in still_used} + rels_name = _Package.rels_name(name) + if unused and rels_name in destination.files: + rels = _parse(destination.files[rels_name]) + for rel in list(rels.iter(_q("rel", "Relationship"))): + if rel.get("Id") in unused: + rels.remove(rel) + destination.files[rels_name] = _serialize(rels) + return removed + + +def _write_sections(destination, sections): + from lxml import etree + + presentation = destination.presentation + extension_list = presentation.find(_q("p", "extLst")) + if extension_list is None: + extension_list = etree.SubElement(presentation, _q("p", "extLst")) + for extension in list(extension_list): + if extension.get("uri") == _SECTION_EXTENSION_URI: + extension_list.remove(extension) + extension = etree.Element(_q("p", "ext"), {"uri": _SECTION_EXTENSION_URI}) + section_list = etree.SubElement(extension, _q("p14", "sectionLst"), nsmap={"p14": _NS["p14"]}) + for position, (name, slide_ids) in enumerate(sections): + if not slide_ids: + continue + # Derived, not random, so the same decks always give the same bytes. + section_id = uuid.uuid5(_SECTION_ID_NAMESPACE, f"{position}:{name}") + section = etree.SubElement(section_list, _q("p14", "section"), { + "name": _section_name(name), "id": "{" + str(section_id).upper() + "}", + }) + ids = etree.SubElement(section, _q("p14", "sldIdLst")) + for slide_id in slide_ids: + etree.SubElement(ids, _q("p14", "sldId"), {"id": str(slide_id)}) + extension_list.insert(0, extension) + + +def _section_name(file_name): + stem = posixpath.splitext(str(file_name or ""))[0].strip() or "Deck" + return stem[:255] + + +def _validate(spool, expected_slides): + # python-pptx re-opens the package, so a broken relationship fails here, not for the user. + from pptx import Presentation + + presentation = Presentation(spool) + if len(presentation.slides) != expected_slides: + raise DocumentMergeError("merge_failed", "The merged deck did not contain every selected slide.") + for slide in presentation.slides: + _ = slide.slide_layout + spool.seek(0) + + +__all__ = ["assemble_pptx"] diff --git a/application/single_app/functions_document_merge_workbook.py b/application/single_app/functions_document_merge_workbook.py new file mode 100644 index 000000000..69ec4c834 --- /dev/null +++ b/application/single_app/functions_document_merge_workbook.py @@ -0,0 +1,364 @@ +# functions_document_merge_workbook.py +"""Workbook assembly for V2 file merge: each source file becomes its own sheet. + +Version: 0.261.248 + +Excel sources keep their cell types — numbers stay numbers and dates stay dates with +their number format — while CSV sources are copied as text so codes keep their leading +zeros. Formulas contribute their last calculated values. Strings that look like formulas +are written as text and never evaluated. Control characters, which Excel can't store, are +removed and counted. Styles, column widths, merged cells, charts and images are not +copied. The workbook carries fixed dates, so the same files give the same bytes. Excel +packages are checked for unsafe XML, with cancellation checks, before they are opened. +""" + +from datetime import date, datetime, time, timedelta +import io +import math +import re +import zipfile + +from functions_document_merge_core import ( + DocumentMergeError, + WORKBOOK_SHEETS_ALL, + finish_output, + guard_ooxml_package, + new_output_spool, + normalized_package, +) + + +_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +_INVALID_SHEET_CHARACTERS = set('[]:*?/\\') +# XML 1.0 can't hold these, so openpyxl refuses them with an error that quotes the cell. +_ILLEGAL_CHARACTERS = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]") +_MAX_SHEET_NAME = 31 +_MAX_CELL_CHARACTERS = 32_767 +_GENERAL_FORMATS = ("General", None, "") +_PACKAGE_DATE = datetime(2000, 1, 1) + + +def assemble_workbook(context): + """Copy each part's selected sheets, in order, into one new workbook.""" + # openpyxl loads only when a workbook merge runs. + from openpyxl import Workbook + + output = Workbook(write_only=True) + try: + return _assemble_into(output, context) + except BaseException: + _discard_workbook(output) + raise + + +def _assemble_into(output, context): + used_names = set() + totals = {"sheets": 0, "cells": 0} + every_sheet = context.options.sheets == WORKBOOK_SHEETS_ALL and context.options.sheet is None + for index, part in enumerate(context.parts): + content = context.load(index) + try: + _copy_part(output, context, index, part, content, used_names, totals, every_sheet) + except DocumentMergeError: + raise + except Exception as exc: + if not context.blames_file(exc): + raise + # Library errors can quote cell text, so only the file's name is reported. + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} couldn't be copied into the workbook.", + ) from exc + context.progress(index) + + context.report["totals"]["sheets"] = totals["sheets"] + context.report["totals"]["cells"] = totals["cells"] + context.limit( + "Cell values and number formats are copied. Styles, column widths, merged cells, charts, images " + "and formulas are not; formulas contribute their last calculated values." + ) + if any(part.resolved_extension() == ".csv" for part in context.parts): + context.limit("CSV files are copied as text, so codes keep their leading zeros.") + return finish_output(context, _write_package(output, context)) + + +def _copy_part(output, context, index, part, content, used_names, totals, every_sheet): + extension = part.resolved_extension() + if extension == ".csv": + sheets = _csv_sheets(part, content) + elif extension == ".xls": + sheets = _xls_sheets(part, content, context) + else: + guard_ooxml_package(content, part, context.limits, "Excel workbook", check=context.check_cancel) + sheets = _xlsx_sheets(part, content, context) + entry = context.part_entry(index) + entry["sheets"] = 0 + entry["rows"] = 0 + removed = 0 + for sheet_label, rows in sheets: + context.check_cancel() + if totals["sheets"] >= context.limits.max_sheets: + raise DocumentMergeError( + "sheet_limit_exceeded", f"The merged workbook would have more than {context.limits.max_sheets} sheets.", + ) + # One sheet per file is named after the file; every-sheet mode adds the sheet name. + name = _unique_sheet_name(_sheet_base_name(part, sheet_label if every_sheet else None), used_names) + worksheet = output.create_sheet(title=name) + written, stripped = _write_rows(worksheet, rows, context, totals, part) + entry["sheets"] += 1 + entry["rows"] += written + entry.setdefault("sheet_names", []).append(name) + totals["sheets"] += 1 + removed += stripped + if entry["sheets"] == 0: + raise DocumentMergeError("empty_source", f"{part.display_name()} has no sheets to merge.") + if removed: + context.warn( + "control_characters_removed", f"{removed:,} control character(s) that Excel can't store were removed.", + index, + ) + + +def _write_package(output, context): + """Write the workbook with fixed dates, so the same files always give the same bytes.""" + from openpyxl.writer.excel import ExcelWriter + + output.properties.creator = "SimpleChat" + output.properties.created = output.properties.modified = _PACKAGE_DATE + package = new_output_spool() + try: + # Workbook.save() would stamp the current time as the modified date. + with zipfile.ZipFile(package, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as archive: + ExcelWriter(output, archive).write_data() + return normalized_package(package, context) + except DocumentMergeError: + raise + except Exception as exc: + if not context.blames_file(exc): + raise + raise DocumentMergeError("merge_failed", "The merged workbook couldn't be written.") from exc + finally: + package.close() + + +def _discard_workbook(workbook): + """Finish and delete the temporary sheet streams of a write-only workbook that won't be saved.""" + for worksheet in list(getattr(workbook, "_sheets", ())): + try: + worksheet.close() + except Exception: + # The workbook is already failing; best-effort cleanup errors are ignored. + pass + writer = getattr(worksheet, "_writer", None) + if writer is not None: + try: + writer.cleanup() + except Exception: + # The workbook is already failing; best-effort cleanup errors are ignored. + pass + + +def _csv_sheets(part, content): + # The merge engine owns CSV encoding and delimiter detection. + from functions_tabular_merge import TabularMergeError, iter_csv_rows + + try: + _encoding, _delimiter, rows = iter_csv_rows(content, part.display_name()) + except TabularMergeError as exc: + raise DocumentMergeError("unreadable_document", exc.message) from exc + + def checked_rows(): + try: + for values in rows: + yield [(value, None) for value in values] + except TabularMergeError as exc: + raise DocumentMergeError("unreadable_document", exc.message) from exc + + yield None, checked_rows() + + +def _xlsx_sheets(part, content, context): + from openpyxl import load_workbook + + try: + workbook = load_workbook(io.BytesIO(content), read_only=True, data_only=True, keep_links=False) + except Exception as exc: + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} couldn't be opened as an Excel workbook.", + ) from exc + try: + names = _chosen_sheets( + part, [(sheet.title, getattr(sheet, "sheet_state", "visible")) for sheet in workbook.worksheets], + context, + ) + for name in names: + worksheet = workbook[name] + worksheet.reset_dimensions() + yield name, _xlsx_rows(worksheet, part) + finally: + workbook.close() + + +def _xlsx_rows(worksheet, part): + try: + for row in worksheet.iter_rows(): + yield [(cell.value, getattr(cell, "number_format", None)) for cell in row] + except DocumentMergeError: + raise + except Exception as exc: + raise DocumentMergeError("unreadable_document", f"{part.display_name()} couldn't be read.") from exc + + +def _xls_sheets(part, content, context): + if content[:8] != _OLE_MAGIC: + raise DocumentMergeError( + "unreadable_document", f"{part.display_name()} isn't a valid Excel 97-2003 (.xls) workbook.", + ) + import xlrd + + try: + book = xlrd.open_workbook(file_contents=content, on_demand=True) + except Exception as exc: + code = "encrypted_document" if "encrypt" in str(exc).lower() else "unreadable_document" + message = ( + f"{part.display_name()} is password-protected or encrypted, so it can't be merged." + if code == "encrypted_document" else f"{part.display_name()} couldn't be opened as an Excel workbook." + ) + raise DocumentMergeError(code, message) from exc + try: + names = book.sheet_names() + # The workbook header records visibility, so no sheet is parsed just to read it. + visibility = getattr(book, "_sheet_visibility", None) + if not isinstance(visibility, list) or len(visibility) != len(names): + visibility = [getattr(book.sheet_by_index(position), "visibility", 0) for position in range(len(names))] + states = [(name, "visible" if state == 0 else "hidden") for name, state in zip(names, visibility)] + for name in _chosen_sheets(part, states, context): + sheet = book.sheet_by_name(name) + yield name, _xls_rows(xlrd, sheet, book.datemode) + # The sheet's rows were written before the next sheet was asked for. + book.unload_sheet(name) + finally: + try: + book.release_resources() + except Exception: + # The reader has finished; release failures do not change the merge result. + pass + + +def _xls_rows(xlrd, sheet, datemode): + for row_index in range(sheet.nrows): + values = [] + for cell in sheet.row(row_index): + if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + values.append((None, None)) + elif cell.ctype == xlrd.XL_CELL_DATE: + try: + moment = xlrd.xldate_as_datetime(cell.value, datemode) + pattern = "yyyy-mm-dd" if moment.time() == time(0, 0) else "yyyy-mm-dd hh:mm:ss" + values.append((moment, pattern)) + except Exception: + values.append((cell.value, None)) + elif cell.ctype == xlrd.XL_CELL_BOOLEAN: + values.append((bool(cell.value), None)) + elif cell.ctype == xlrd.XL_CELL_ERROR: + values.append((xlrd.error_text_from_code.get(cell.value, "#ERROR"), None)) + else: + values.append((cell.value, None)) + yield values + + +def _chosen_sheets(part, sheets, context): + names = [name for name, _ in sheets] + if not names: + raise DocumentMergeError("empty_source", f"{part.display_name()} has no worksheets.") + requested = context.options.sheet + if requested is not None: + for name in names: + if name == requested or name.casefold() == requested.casefold(): + return [name] + raise DocumentMergeError( + "sheet_not_found", f'{part.display_name()} has no sheet named "{requested}".', + details={"available_sheets": names}, + ) + visible = [name for name, state in sheets if state == "visible"] or names[:1] + if context.options.sheets == WORKBOOK_SHEETS_ALL: + hidden = len(names) - len(visible) + if hidden: + context.warn("hidden_sheets_skipped", f"{hidden} hidden sheet(s) were not copied.") + return visible + return visible[:1] + + +def _write_rows(worksheet, rows, context, totals, part): + """Append the rows; return how many were written and how many characters were removed.""" + from openpyxl.cell import WriteOnlyCell + + written = 0 + removed = 0 + for values in rows: + if written >= context.limits.max_sheet_rows: + raise DocumentMergeError( + "row_limit_exceeded", f"A sheet from {part.display_name()} has more rows than Excel allows.", + ) + totals["cells"] += len(values) + if totals["cells"] > context.limits.max_total_cells: + raise DocumentMergeError( + "cell_limit_exceeded", f"The merged workbook would have more than {context.limits.max_total_cells:,} cells.", + ) + cells = [] + for value, number_format in values: + value, stripped = _cell_value(value, part) + removed += stripped + cell = WriteOnlyCell(worksheet, value=value) + if isinstance(value, str): + # Literal text, never a formula, even when it starts with "=". + cell.data_type = "s" + elif number_format not in _GENERAL_FORMATS and isinstance(value, (int, float, datetime, date, time)): + cell.number_format = number_format + cells.append(cell) + worksheet.append(cells) + written += 1 + if written % 1000 == 0: + context.check_cancel() + return written, removed + + +def _cell_value(value, part): + """The value to write, and how many characters Excel can't store were removed from it.""" + if value is None or isinstance(value, (bool, int, float, datetime, date, time, timedelta)): + if isinstance(value, float) and math.isnan(value): + return None, 0 + return value, 0 + text, removed = _ILLEGAL_CHARACTERS.subn("", str(value)) + if len(text) > _MAX_CELL_CHARACTERS: + raise DocumentMergeError( + "cell_too_large", f"{part.display_name()} has a value longer than an Excel cell allows.", + ) + return text, removed + + +def _sheet_base_name(part, sheet_label): + stem = part.display_name() + for extension in (".xlsx", ".xlsm", ".xls", ".csv"): + if stem.lower().endswith(extension): + stem = stem[: -len(extension)] + break + if sheet_label: + stem = f"{stem} - {sheet_label}" + return stem + + +def _unique_sheet_name(base, used_names): + cleaned = _ILLEGAL_CHARACTERS.sub(" ", str(base)) + cleaned = "".join(" " if character in _INVALID_SHEET_CHARACTERS else character for character in cleaned) + cleaned = " ".join(cleaned.split()).strip("'").strip() or "Sheet" + candidate = cleaned[:_MAX_SHEET_NAME].strip("'").strip() or "Sheet" + suffix = 2 + while candidate.casefold() in used_names: + tail = f" ({suffix})" + candidate = cleaned[: _MAX_SHEET_NAME - len(tail)].strip("'").strip() + tail + suffix += 1 + used_names.add(candidate.casefold()) + return candidate + + +__all__ = ["assemble_workbook"] diff --git a/application/single_app/functions_generated_export_registry.py b/application/single_app/functions_generated_export_registry.py index 0942a99f2..ce7d3e6e7 100644 --- a/application/single_app/functions_generated_export_registry.py +++ b/application/single_app/functions_generated_export_registry.py @@ -37,6 +37,12 @@ class GeneratedFileExportFormat: PREPARED_SLIDE_DECK_VERSION = 'prepared_slide_deck_v1' +EXACT_TABULAR_RECORDS_PROFILE = 'exact_tabular_records_v1' +EXACT_TABULAR_WORKBOOK_PROFILE = 'exact_tabular_workbook_v1' +EXACT_TABULAR_PROFILES = (EXACT_TABULAR_RECORDS_PROFILE, EXACT_TABULAR_WORKBOOK_PROFILE) +# A file assembled again from the original files a document merge described and checked. +ASSEMBLED_DOCUMENT_PROFILE = 'assembled_document_v1' +EXACT_TABULAR_DEFAULT_SHEET_NAME = 'Sheet1' GENERATED_IMAGE_REFERENCE_PATTERN = r'^asset:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' _OPTION_SCHEMAS = { 'columns': { @@ -158,10 +164,15 @@ class GeneratedFileExportFormat: GeneratedFileExportProfile('structured_records_v1', ('records',)), GeneratedFileExportProfile('structured_value_v1', ('structured_value',)), ) +_ASSEMBLED_PROFILE = GeneratedFileExportProfile(ASSEMBLED_DOCUMENT_PROFILE, ('structured_value',)) GENERATED_FILE_EXPORT_REGISTRY = ( GeneratedFileExportFormat( 'csv', ('csv',), 'csv', 'text/csv; charset=utf-8', - (GeneratedFileExportProfile('tabular_records_v1', ('records',), ('columns',)),), + ( + GeneratedFileExportProfile('tabular_records_v1', ('records',), ('columns',)), + # Every retained column in retained order, for results whose schema is known only at run time. + GeneratedFileExportProfile(EXACT_TABULAR_RECORDS_PROFILE, ('records',)), + ), 'csv', ), GeneratedFileExportFormat( @@ -190,24 +201,34 @@ class GeneratedFileExportFormat: ), GeneratedFileExportFormat( 'xlsx', ('xlsx',), 'xlsx', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - (GeneratedFileExportProfile('tabular_workbook_v1', ('records',), ('columns', 'sheet_name')),), + ( + GeneratedFileExportProfile('tabular_workbook_v1', ('records',), ('columns', 'sheet_name')), + GeneratedFileExportProfile(EXACT_TABULAR_WORKBOOK_PROFILE, ('records',), optional_options=('sheet_name',)), + _ASSEMBLED_PROFILE, + ), 'office', ('openpyxl',), ), GeneratedFileExportFormat( 'docx', ('docx',), 'docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', - (GeneratedFileExportProfile('prepared_report_v1', ('text', 'markdown'), optional_options=('title',)),), + ( + GeneratedFileExportProfile('prepared_report_v1', ('text', 'markdown'), optional_options=('title',)), + _ASSEMBLED_PROFILE, + ), 'office', ('python-docx', 'markdown2', 'beautifulsoup4', 'Pillow'), streaming=False, rich_media=True, ), GeneratedFileExportFormat( 'pdf', ('pdf',), 'pdf', 'application/pdf', - (GeneratedFileExportProfile('prepared_report_v1', ('text', 'markdown'), optional_options=('title',)),), + ( + GeneratedFileExportProfile('prepared_report_v1', ('text', 'markdown'), optional_options=('title',)), + _ASSEMBLED_PROFILE, + ), 'office', ('PyMuPDF', 'markdown2', 'beautifulsoup4', 'Pillow'), streaming=False, rich_media=True, ), GeneratedFileExportFormat( 'pptx', ('pptx',), 'pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', - (GeneratedFileExportProfile(PREPARED_SLIDE_DECK_VERSION, ('structured_value',)),), + (GeneratedFileExportProfile(PREPARED_SLIDE_DECK_VERSION, ('structured_value',)), _ASSEMBLED_PROFILE), 'office', ('python-pptx', 'PyMuPDF', 'Pillow'), streaming=False, rich_media=True, ), @@ -237,11 +258,14 @@ def _options_schema(profile): def _input_schema(profile): if profile.profile == PREPARED_SLIDE_DECK_VERSION: return get_prepared_slide_deck_schema() + if profile.profile == ASSEMBLED_DOCUMENT_PROFILE: + # Only a document merge's retained "assembly" output; Render checks it in full. + return {'type': 'object'} if profile.profile in ('prepared_text_v1', 'prepared_report_v1'): return {'type': 'string'} if profile.source_kinds == ('records',): record = {'type': 'object'} - if profile.profile in ('tabular_records_v1', 'tabular_workbook_v1'): + if profile.profile in ('tabular_records_v1', 'tabular_workbook_v1', *EXACT_TABULAR_PROFILES): record['additionalProperties'] = {'type': ['string', 'number', 'boolean', 'null']} return {'type': 'array', 'items': record} return {} diff --git a/application/single_app/functions_generated_file_exports.py b/application/single_app/functions_generated_file_exports.py index 6cb3cbdce..694cbfc06 100644 --- a/application/single_app/functions_generated_file_exports.py +++ b/application/single_app/functions_generated_file_exports.py @@ -678,6 +678,7 @@ def build_generated_file_export( limits: Optional[GeneratedFileExportLimits] = None, office_limits: Optional["OfficeRenderLimits"] = None, image_resolver: Optional[Callable[[str], bytes]] = None, + document_reader: Optional[Callable[[str], bytes]] = None, ) -> GeneratedFileExportStream: ... @@ -707,6 +708,7 @@ def build_generated_file_export( limits: Optional[GeneratedFileExportLimits] = None, office_limits: Optional["OfficeRenderLimits"] = None, image_resolver: Optional[Callable[[str], bytes]] = None, + document_reader: Optional[Callable[[str], bytes]] = None, ) -> Optional[Dict[str, Any]] | GeneratedFileExportStream: """Render an explicit complete source, or preserve the existing response-export policy.""" if source is not None or export_request is not None: @@ -723,14 +725,14 @@ def build_generated_file_export( return _render_generated_office_source( source, export_request, max_output_bytes=max_output_bytes, check=check, limits=limits, - office_limits=office_limits, image_resolver=image_resolver, + office_limits=office_limits, image_resolver=image_resolver, document_reader=document_reader, ) - if office_limits is not None or image_resolver is not None: + if office_limits is not None or image_resolver is not None or document_reader is not None: raise GeneratedFileExportError('invalid_options', 'This export profile does not accept Office options.') return _render_generated_file_source( source, export_request, max_output_bytes=max_output_bytes, check=check, limits=limits, ) - if office_limits is not None or image_resolver is not None: + if office_limits is not None or image_resolver is not None or document_reader is not None: raise GeneratedFileExportError('invalid_options', 'Office options require an explicit Office source request.') output_format = get_requested_generated_file_format(user_question) or _normalize_pending_output_format( pending_output_format, diff --git a/application/single_app/functions_generated_office_adapters.py b/application/single_app/functions_generated_office_adapters.py index 768e42bc6..fe5c6cc49 100644 --- a/application/single_app/functions_generated_office_adapters.py +++ b/application/single_app/functions_generated_office_adapters.py @@ -18,6 +18,9 @@ GeneratedFileExportStream, ) from functions_generated_export_registry import ( + ASSEMBLED_DOCUMENT_PROFILE, + EXACT_TABULAR_DEFAULT_SHEET_NAME, + EXACT_TABULAR_WORKBOOK_PROFILE, GENERATED_IMAGE_REFERENCE_PATTERN, PREPARED_SLIDE_DECK_VERSION, get_prepared_slide_deck_schema, @@ -29,6 +32,7 @@ _SourceChecks, _iter_records, _render_text, + _tabular_request_columns, _validate_column_names, _validate_export_limits, _validate_source_limits, @@ -252,15 +256,55 @@ def resolve(reference): return resolve +def _render_assembled_document(source, request, entry, *, max_output_bytes, check, limits, document_reader): + """Assemble a merged file again from its original files and deliver it only if identical.""" + # The merge engine and its assemblers load only when a merged file is rendered. + from functions_document_merge_assembly import DocumentAssemblyError, assembled_output_format, reassemble_document + + if not callable(document_reader): + raise GeneratedFileExportError('unsupported_source', 'A merged file needs its original files.') + limits = GeneratedFileExportLimits() if limits is None else limits + _validate_source_limits(source, limits, max_output_bytes, check) + checks = _SourceChecks(source, check) + checks.run() + value = source.read_value() + try: + if assembled_output_format(value) != entry.format_id: + raise GeneratedFileExportError('unsupported_source', 'This merged file is a different format.') + merged, content_sha256 = reassemble_document( + value, document_reader, max_output_bytes=max_output_bytes, check=checks.run, + ) + except DocumentAssemblyError as exc: + raise GeneratedFileExportError(exc.code, exc.message) from exc + try: + checks.run() + return GeneratedFileExportStream( + file_content=merged.open_stream(), output_format=entry.format_id, media_type=entry.media_type, + size_bytes=merged.size_bytes, content_sha256=content_sha256, record_count=0, + profile=request.profile, source_kind=source.kind, file_extension=entry.file_extension, + character_count=None, metadata={'renderer_profile': request.profile}, + ) + except BaseException: + merged.close() + raise + + def _render_generated_office_source( source, request, *, max_output_bytes, check=None, limits=None, office_limits=None, - image_resolver=None, + image_resolver=None, document_reader=None, ): entry = resolve_generated_file_export_format(request, getattr(source, 'kind', None)) if entry.renderer_id != 'office': raise GeneratedFileExportError('unsupported_format', 'This Office export format is not supported.') if image_resolver is not None and (not entry.rich_media or not callable(image_resolver)): raise GeneratedFileExportError('invalid_options', 'This export does not accept the supplied image resolver.') + if request.profile == ASSEMBLED_DOCUMENT_PROFILE: + return _render_assembled_document( + source, request, entry, max_output_bytes=max_output_bytes, check=check, limits=limits, + document_reader=document_reader, + ) + if document_reader is not None: + raise GeneratedFileExportError('invalid_options', 'Only a merged file reads original files.') limits = GeneratedFileExportLimits() if limits is None else limits _validate_source_limits(source, limits, max_output_bytes, check) renderer_limits = _office_limits(office_limits, limits, max_output_bytes) @@ -272,12 +316,17 @@ def _render_generated_office_source( rows = None with ExitStack() as resources: if entry.format_id == 'xlsx': - columns = _validate_column_names(request.columns, limits, checks, label='XLSX') + columns = _validate_column_names( + _tabular_request_columns(source, request), limits, checks, label='XLSX', + ) + sheet_name = request.sheet_name + if sheet_name is None and request.profile == EXACT_TABULAR_WORKBOOK_PROFILE: + sheet_name = EXACT_TABULAR_DEFAULT_SHEET_NAME rows = resources.enter_context(ClosingExportResource( boundary.rows(_workbook_records(source, request, columns, limits, checks)), )) prepared = office.PreparedWorkbook(( - office.PreparedSheet(request.sheet_name, columns, rows, checks.expected_count), + office.PreparedSheet(sheet_name, columns, rows, checks.expected_count), )) renderer = office.render_prepared_xlsx elif entry.format_id in ('docx', 'pdf'): diff --git a/application/single_app/functions_ooxml_package_guard.py b/application/single_app/functions_ooxml_package_guard.py new file mode 100644 index 000000000..e80eff3c5 --- /dev/null +++ b/application/single_app/functions_ooxml_package_guard.py @@ -0,0 +1,219 @@ +# functions_ooxml_package_guard.py +"""Checks of an Office Open XML package's XML parts before any library parses them. + +Version: 0.261.245 +Implemented in: 0.261.245 + +Office Open XML requires its XML parts to be UTF-8 or UTF-16 and forbids document type +declarations in them, and Office refuses files that break either rule. A declaration is +how a hostile part asks an XML parser to read other files or to expand entities without +bound, and a part written or declared in another encoding could hide one, so file merges +refuse such packages before any library parses them, even though the parsers they use +don't resolve entities. + +A part is checked when its name ends in ``.xml``, ``.rels`` or ``.vml``, or when the +package's ``[Content_Types].xml`` declares it an Office XML type, because Office and the +libraries choose how to read a part by its declared type, not its name. Pictures and web +pages, such as SVG images or imported HTML, may legitimately declare a document type and +are not checked. Only the prolog of each part is read: at most 4 KiB per part, read in +growing chunks, and at most 32 MiB for the whole package. Standard library only. +""" + +import re +from xml.etree import ElementTree +import zipfile +import zlib + +try: + # zipfile imports lzma already when it is available. + import lzma + _LZMA_ERRORS = (lzma.LZMAError,) +except ImportError: # pragma: no cover - only on Python builds without lzma + _LZMA_ERRORS = () + + +XML_PART_SUFFIXES = (".xml", ".rels", ".vml") +CONTENT_TYPES_PART = "[content_types].xml" +_GENERIC_XML_TYPES = frozenset({"application/xml", "text/xml"}) +# Errors reading a damaged, encrypted or inconsistent ZIP entry, or an unreadable part list. The +# package is in memory, so an OSError here comes from a decompressor, not from storage. +_READ_ERRORS = ( + zipfile.BadZipFile, zipfile.LargeZipFile, RuntimeError, NotImplementedError, EOFError, OSError, + zlib.error, ElementTree.ParseError, UnicodeError, +) + _LZMA_ERRORS +_FIRST_READ_BYTES = 256 +_PROLOG_BYTES = 4 * 1024 +_PACKAGE_READ_BUDGET = 32 * 1024 * 1024 +_MAX_CONTENT_TYPES_BYTES = 16 * 1024 * 1024 +_CHECK_EVERY_PARTS = 64 +# Whitespace, the XML declaration, processing instructions and comments may come before the root element. +_PROLOG = re.compile(r"\ufeff?(?:\s+|<\?.*?\?>|)*", re.DOTALL) +_DECLARATION = re.compile(r"\ufeff?<\?xml\s(.*?)\?>", re.DOTALL) +_DECLARED_ENCODING = re.compile(r"""encoding\s*=\s*["']([^"']*)["']""") +# Office Open XML parts must be UTF-8 or UTF-16. A part that declares another encoding, such as +# UTF-7, could read one way here and another way in a parser. +_ALLOWED_ENCODINGS = frozenset({"utf-8", "utf8", "utf-16", "utf-16le", "utf-16be"}) + + +class UnreadablePackageError(ValueError): + """A part of the package, or its list of content types, couldn't be read.""" + + +def _encoding(head): + if head.startswith(b"\xff\xfe"): + return "utf-16-le" + if head.startswith(b"\xfe\xff"): + return "utf-16-be" + if len(head) >= 2 and head[0] != 0 and head[1] == 0: + return "utf-16-le" + if len(head) >= 2 and head[0] == 0 and head[1] != 0: + return "utf-16-be" + return "utf-8" + + +def _verdict(head, final): + """True to refuse, False once the root element starts, None when more is needed. + + With ``final`` the whole part has been read, so the answer is never None. + """ + text = head.decode(_encoding(head), errors="replace") + declaration = _DECLARATION.match(text) + if declaration is not None: + encoding = _DECLARED_ENCODING.search(declaration.group(1)) + if encoding is not None and encoding.group(1).strip().lower() not in _ALLOWED_ENCODINGS: + return True + rest = text[_PROLOG.match(text).end():] + if not rest: + # Empty, or only whitespace, comments and instructions so far; nothing to parse yet. + return False if final else None + if rest[0] != "<": + # Not UTF-8 or UTF-16 XML, which Office Open XML requires. + return True + if len(rest) == 1: + return True if final else None + marker = rest[1] + if marker == "!": + if rest.startswith("= _PROLOG_BYTES: + return True + size = min(size * 2, _PROLOG_BYTES) + + +def is_office_xml_part(name, content_type): + """Whether Office or a library reads the part as Office XML, by its name or declared type.""" + if name.lower().endswith(XML_PART_SUFFIXES): + return True + content_type = (content_type or "").strip().lower() + if content_type in _GENERIC_XML_TYPES: + return True + return content_type.startswith("application/vnd.") and ( + content_type.endswith("+xml") or content_type.endswith(".vmldrawing") + ) + + +def _declared_content_types(archive, entry, names): + """Extension defaults and per-part overrides, lowercased, for the package's own parts.""" + defaults, overrides = {}, {} + with archive.open(entry) as stream: + # The prolog was checked first, so the list declares no entities. + for _event, element in ElementTree.iterparse(stream): + tag = element.tag.rsplit("}", 1)[-1] + if tag == "Default": + extension = (element.get("Extension") or "").strip().lower() + if extension: + defaults[extension] = element.get("ContentType") or "" + elif tag == "Override": + part_name = (element.get("PartName") or "").strip().lstrip("/").lower() + if part_name in names: + overrides[part_name] = element.get("ContentType") or "" + element.clear() + return defaults, overrides + + +def content_type_of(name, defaults, overrides): + """A part's declared content type, matched without regard to case as Office does.""" + lowered = name.lower() + if lowered in overrides: + return overrides[lowered] + base = lowered.rsplit("/", 1)[-1] + return defaults.get(base.rpartition(".")[2] if "." in base else "", "") + + +def first_unsafe_xml_part(archive, entries, *, check=None): + """The name of the first XML part with a document type declaration or unreadable XML, or None. + + ``check()`` is called between parts and its errors pass through unchanged. ZIP read + errors and an unreadable part list raise ``UnreadablePackageError``. + """ + budget = _Budget(_PACKAGE_READ_BUDGET) + files = [entry for entry in entries if not entry.is_dir()] + names = {entry.filename.lower() for entry in files} + defaults, overrides = {}, {} + listed = next((entry for entry in files if entry.filename.lower() == CONTENT_TYPES_PART), None) + if listed is not None: + if listed.file_size > _MAX_CONTENT_TYPES_BYTES: + return listed.filename + try: + if _unsafe_prolog(archive, listed, budget): + return listed.filename + defaults, overrides = _declared_content_types(archive, listed, names) + except _READ_ERRORS as exc: + raise UnreadablePackageError(listed.filename) from exc + for position, entry in enumerate(files): + if check is not None and position % _CHECK_EVERY_PARTS == 0: + check() + if entry is listed or not is_office_xml_part(entry.filename, content_type_of(entry.filename, defaults, overrides)): + continue + try: + unsafe = _unsafe_prolog(archive, entry, budget) + except _READ_ERRORS as exc: + raise UnreadablePackageError(entry.filename) from exc + if unsafe: + return entry.filename + return None + + +__all__ = [ + "CONTENT_TYPES_PART", + "UnreadablePackageError", + "XML_PART_SUFFIXES", + "content_type_of", + "first_unsafe_xml_part", + "is_office_xml_part", +] diff --git a/application/single_app/functions_orchestration_adapters.py b/application/single_app/functions_orchestration_adapters.py index 48385110c..ab1b1e965 100644 --- a/application/single_app/functions_orchestration_adapters.py +++ b/application/single_app/functions_orchestration_adapters.py @@ -86,8 +86,11 @@ CAPABILITY_DEEP_RESEARCH, CAPABILITY_DOCUMENT_ANALYZE, CAPABILITY_DOCUMENT_COMPARE, + CAPABILITY_DOCUMENT_MERGE, CAPABILITY_DOCUMENT_SEARCH, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_INSPECT, + CAPABILITY_TABULAR_MERGE, CAPABILITY_URL_FETCH, CAPABILITY_WEB_SEARCH, CAPABILITY_WORKFLOW_PROPOSE, @@ -105,6 +108,7 @@ STEP_STATUS_COMPLETED, STEP_STATUS_FAILED, STEP_STATUS_PARTIAL, + access_failure, build_step_result, build_failure, failure_from_exception, @@ -948,6 +952,240 @@ def run_document_compare(step, context, *, settings, user_id, emit, cancel_reque ) +# -------------------------------------------------------------------------------------- +# tabular_inspect and tabular_merge -> functions_tabular_merge through functions_orchestration_merge +# -------------------------------------------------------------------------------------- + +def _tabular_step_failure(exc, step, user_id, *, failure_code, label, error_type=None): + """The result for a refused or failed merge step; only catalog text reaches the user.""" + if error_type is None: + from functions_tabular_merge import TabularMergeError as error_type + + if isinstance(exc, error_type): + failure = build_failure(failure_code(exc)) + log_event( + f'{_LOG_PREFIX} {label} was refused.', + extra={ + 'user_id': user_id, 'step_id': (step or {}).get('step_id'), + 'merge_code': exc.code, 'failure_code': failure['code'], + }, + level=logging.WARNING, + ) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + # Typed authority, cancellation and lifecycle controls belong to the owning runtime. + from functions_orchestration_result_runtime import raise_source_service_failure + + raise_source_service_failure(exc) + log_event( + f'{_LOG_PREFIX} {label} failed.', + extra={'user_id': user_id, 'step_id': (step or {}).get('step_id'), 'error_type': type(exc).__name__}, + level=logging.ERROR, exceptionTraceback=True, + ) + if isinstance(exc, (PermissionError, LookupError)) or type(exc).__name__ in ( + 'ResultUnavailableError', 'DocumentHeldError', + ): + failure = access_failure(exc) + return build_step_result( + status=STEP_STATUS_FAILED, summary=failure['message'], error=failure['message'], failure=failure, + ) + return _failed_result(f'The {label.lower()} could not be completed.', exc) + + +def run_tabular_inspect(step, context, *, settings, user_id, emit, cancel_requested): + """Describe authorized spreadsheets and how their columns line up; no model and no file.""" + arguments = _arguments(step) + # The merge engine is standard-library only; its Excel readers load only when used. + from functions_tabular_merge import TabularMergeCancelled, TabularMergeError + + try: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect') + input_fingerprint = _internal_result_input_fingerprint(step, context) + from functions_orchestration_merge import ( + build_tabular_merge_sources, + inspection_step_summary, + persist_tabular_inspection_result, + require_tabular_merge_manifest, + tabular_inspect_options, + tabular_merge_limits, + ) + from functions_tabular_merge import inspect_tabular_sources + + with orchestration_file_policy(allow_generated_files=False): + document_ids = _string_list(arguments.get('document_ids')) + limits = tabular_merge_limits(settings) + if len(document_ids) > limits.max_sources: + # A bound source set is checked here; explicit IDs were checked at planning. + raise TabularMergeError('too_many_sources', 'Too many files were selected to inspect here.') + options = tabular_inspect_options(arguments) + service, producer, token, input_fingerprint = _internal_result_context( + step, context, user_id, CAPABILITY_TABULAR_INSPECT, input_fingerprint=input_fingerprint, + ) + manifest = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, + ) + require_tabular_merge_manifest(manifest, document_ids, minimum=1) + _emit(emit, _progress(step, CAPABILITY_TABULAR_INSPECT, 'Inspecting spreadsheets')) + sources = build_tabular_merge_sources( + manifest, user_id, byte_reader=_ctx(context, 'merge_source_reader', None), + ) + inspection = inspect_tabular_sources( + sources, options=options, limits=limits, + cancel_requested=lambda: _is_cancelled(cancel_requested), + ) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect_saving') + task_result = persist_tabular_inspection_result( + service=service, producer=producer, inspection=inspection, sources=manifest, + guard_token=token, input_fingerprint=input_fingerprint, + ) + summary = inspection_step_summary(inspection) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_inspect_finalization') + except (MixedSourceCancellationError, TabularMergeCancelled): + return _cancelled_result('Inspecting spreadsheets was cancelled.') + except Exception as exc: + from functions_orchestration_merge import inspect_failure_code + + return _tabular_step_failure( + exc, step, user_id, failure_code=inspect_failure_code, label='Spreadsheet inspection', + ) + return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) + + +def run_tabular_merge(step, context, *, settings, user_id, emit, cancel_requested): + """Append the rows of authorized spreadsheets into one table; no model and no file. + + A bound ``mapping`` input is a compose step's prepared column mapping; it is validated + again here and makes the merge keep exactly its columns. The retained ``records`` output + is what ``render_file`` turns into a CSV or XLSX file. + """ + arguments = _arguments(step) + # The merge engine is standard-library only; its Excel readers load only when used. + from functions_tabular_merge import TabularMergeCancelled, TabularMergeError + + try: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge') + input_fingerprint = _internal_result_input_fingerprint(step, context) + from functions_orchestration_merge import ( + build_tabular_merge_sources, + merge_step_summary, + persist_tabular_merge_result, + require_tabular_merge_manifest, + tabular_merge_limits, + tabular_merge_options, + ) + from functions_tabular_merge import merge_tabular_sources, tabular_mapping_from_profile + + with orchestration_file_policy(allow_generated_files=False): + document_ids = _string_list(arguments.get('document_ids')) + limits = tabular_merge_limits(settings) + if len(document_ids) > limits.max_sources: + # A bound source set is checked here; explicit IDs were checked at planning. + raise TabularMergeError('too_many_sources', 'Too many files were selected to merge here.') + service, producer, token, input_fingerprint = _internal_result_context( + step, context, user_id, CAPABILITY_TABULAR_MERGE, input_fingerprint=input_fingerprint, + ) + mapping = None + if 'mapping' in (step.get('inputs') or {}): + # Retained-result readers load only when a plan binds a prepared mapping. + from functions_orchestration_result_runtime import read_complete_input, resolve_step_inputs + + readers = resolve_step_inputs(step, context) + mapping = tabular_mapping_from_profile(read_complete_input(readers['mapping'])) + options = tabular_merge_options(arguments, mapping) + manifest = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, + ) + require_tabular_merge_manifest(manifest, document_ids) + _emit(emit, _progress(step, CAPABILITY_TABULAR_MERGE, 'Merging spreadsheets')) + sources = build_tabular_merge_sources( + manifest, user_id, byte_reader=_ctx(context, 'merge_source_reader', None), + ) + with merge_tabular_sources( + sources, options=options, limits=limits, + cancel_requested=lambda: _is_cancelled(cancel_requested), + ) as merge_result: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_saving') + task_result = persist_tabular_merge_result( + service=service, producer=producer, merge_result=merge_result, sources=manifest, + guard_token=token, input_fingerprint=input_fingerprint, mapping=mapping, + ) + summary = merge_step_summary(merge_result) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_merge_finalization') + except (MixedSourceCancellationError, TabularMergeCancelled): + return _cancelled_result('Merging spreadsheets was cancelled.') + except Exception as exc: + from functions_orchestration_merge import merge_failure_code + + return _tabular_step_failure(exc, step, user_id, failure_code=merge_failure_code, label='Spreadsheet merge') + return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) + + +def run_document_merge(step, context, *, settings, user_id, emit, cancel_requested): + """Assemble authorized documents once to check them; no model and no file. + + The retained ``assembly`` output names the files, every option and the checked file's + size and SHA-256. ``render_file`` assembles the same files again with the + ``assembled_document_v1`` profile and delivers the file only when it is identical. + """ + arguments = _arguments(step) + # The merge engine is standard-library only; each assembler loads only when its kind runs. + from functions_document_merge import DocumentMergeCancelled, DocumentMergeError + + try: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_document_merge') + input_fingerprint = _internal_result_input_fingerprint(step, context) + from functions_document_merge import merge_documents + from functions_orchestration_document_merge import ( + document_merge_limits, + document_merge_options_from_arguments, + document_merge_step_summary, + persist_document_merge_result, + require_document_merge_manifest, + ) + from functions_orchestration_merge import build_document_merge_parts + + with orchestration_file_policy(allow_generated_files=False): + document_ids = _string_list(arguments.get('document_ids')) + limits = document_merge_limits(settings) + if len(document_ids) > limits.max_parts: + # A bound source set is checked here; explicit IDs were checked at planning. + raise DocumentMergeError('too_many_sources', 'Too many files were selected to merge here.') + kind, options = document_merge_options_from_arguments(arguments) + service, producer, token, input_fingerprint = _internal_result_context( + step, context, user_id, CAPABILITY_DOCUMENT_MERGE, input_fingerprint=input_fingerprint, + ) + manifest = resolve_context_source_manifest( + context, document_ids, settings=settings, user_id=user_id, cancel_requested=cancel_requested, + ) + require_document_merge_manifest(manifest, document_ids, kind) + _emit(emit, _progress(step, CAPABILITY_DOCUMENT_MERGE, 'Merging documents')) + parts = build_document_merge_parts( + manifest, user_id, byte_reader=_ctx(context, 'merge_source_reader', None), + ) + with merge_documents( + kind, parts, options=options, limits=limits, + cancel_requested=lambda: _is_cancelled(cancel_requested), + ) as merged: + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_document_merge_saving') + task_result = persist_document_merge_result( + service=service, producer=producer, kind=kind, options=options, parts=parts, + merge_result=merged, sources=manifest, guard_token=token, input_fingerprint=input_fingerprint, + ) + summary = document_merge_step_summary(kind, merged.report) + raise_if_mixed_source_cancelled(cancel_requested, 'orchestration_document_merge_finalization') + except (MixedSourceCancellationError, DocumentMergeCancelled): + return _cancelled_result('Merging documents was cancelled.') + except Exception as exc: + from functions_orchestration_document_merge import document_merge_failure_code + + return _tabular_step_failure( + exc, step, user_id, failure_code=document_merge_failure_code, label='Document merge', + error_type=DocumentMergeError, + ) + return build_step_result(status=STEP_STATUS_COMPLETED, summary=summary, task_result=task_result) + + # -------------------------------------------------------------------------------------- # tabular_analyze -> the native tabular bridge the server binds for each step # -------------------------------------------------------------------------------------- @@ -1908,6 +2146,9 @@ def run_workflow_results(step, context, *, settings, user_id, emit=None, cancel_ CAPABILITY_DOCUMENT_ANALYZE: run_document_analyze, CAPABILITY_DOCUMENT_COMPARE: run_document_compare, CAPABILITY_TABULAR_ANALYZE: run_tabular_analyze, + CAPABILITY_TABULAR_INSPECT: run_tabular_inspect, + CAPABILITY_TABULAR_MERGE: run_tabular_merge, + CAPABILITY_DOCUMENT_MERGE: run_document_merge, CAPABILITY_WEB_SEARCH: run_web_search, CAPABILITY_URL_FETCH: run_url_fetch, CAPABILITY_DEEP_RESEARCH: run_deep_research, diff --git a/application/single_app/functions_orchestration_bootstrap.py b/application/single_app/functions_orchestration_bootstrap.py index 968a232a5..899847609 100644 --- a/application/single_app/functions_orchestration_bootstrap.py +++ b/application/single_app/functions_orchestration_bootstrap.py @@ -1,13 +1,14 @@ # functions_orchestration_bootstrap.py """Application-owned factories shared by web requests and scheduler continuations. -Version: 0.261.209 +Version: 0.261.245 Unlike the result/rendering services, this is an application composition root. Import it only after config has initialized the existing clients. Registering the artifact factory performs no I/O; each use rebuilds current actor/source access. External-source identity trusts the signed-in session's app roles, like classic -chat, and makes no Microsoft Graph calls (0.261.209). +chat, and makes no Microsoft Graph calls (0.261.209). Merged documents are rendered +from their own original files through a screening-aware reader (0.261.245). """ import hashlib @@ -254,6 +255,29 @@ def _authorize_render_output(record, *, operation, rendering_service): return True +def build_merge_document_reader(user_id): + """Read one workspace document a merged file was assembled from, with current access. + + The rendering service names only the merge result's own lineage sources and checks the + re-assembled bytes; this reader applies the same screening-aware access as the merge. + """ + def read(source): + # Screening-aware reads load only when a merged file is rendered. + from content_screening.access import read_available_document_bytes + + scope = source.get("scope") + scope_id = source.get("scope_id") + _document, content = read_available_document_bytes( + source, user_id, + scope_id if scope == "group" else None, + scope_id if scope == "public" else None, + purpose="native", + ) + return content + + return read + + def build_image_asset_reader(user_id, conversation_id): """Read a retained generated image's bytes from its own conversation image message. @@ -377,6 +401,7 @@ def authorize_output(record, *, operation): max_output_bytes=min(maximum_mb, 500) * 1024 * 1024, native_bridge_for_step=native_bridge_for_step, image_asset_reader=build_image_asset_reader(user_id, conversation_id), + document_bytes_reader=build_merge_document_reader(user_id), ) return services diff --git a/application/single_app/functions_orchestration_deliverables.py b/application/single_app/functions_orchestration_deliverables.py index dc13c7d31..637bb3f10 100644 --- a/application/single_app/functions_orchestration_deliverables.py +++ b/application/single_app/functions_orchestration_deliverables.py @@ -1,8 +1,10 @@ # functions_orchestration_deliverables.py """What the user asked to receive, and whether the plan can actually deliver each part. -Version: 0.261.141 +Version: 0.261.246 Implemented in: 0.261.138 +Document merge recipes added in: 0.261.245 +One-sheet or sheet-per-file question for spreadsheet merges added in: 0.261.246 The planner lists its plan's deliverables first: the answer, files, images, charts, and diagrams the user asked for (``requested: explicit``) and anything it adds on its own @@ -39,12 +41,16 @@ CAPABILITY_COMPOSE, CAPABILITY_DOCUMENT_ANALYZE, CAPABILITY_DOCUMENT_COMPARE, + CAPABILITY_DOCUMENT_MERGE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_RENDER_FILE, + CAPABILITY_TABULAR_INSPECT, + CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, MAX_GENERATED_IMAGES_PER_PLAN, + TABULAR_COLUMN_MAPPING_PROFILE, VISUAL_CHART, VISUAL_DIAGRAM, VISUAL_IMAGE_PROPOSAL, @@ -79,6 +85,22 @@ MAX_DESCRIPTION_LENGTH = 300 MAX_FORMAT_LENGTH = 40 AI_ILLUSTRATION_LABEL = 'AI-generated image' +# The two answers to "one sheet or a sheet per file?" for a spreadsheet merge into one Excel file. +MERGE_LAYOUT_ONE_SHEET = 'All rows on one sheet' +MERGE_LAYOUT_SHEET_PER_FILE = 'Each file on its own sheet' +MERGE_LAYOUT_FACT = ( + 'Several CSV or Excel files merged into one Excel file can mean every row on one sheet ' + '(tabular_merge, then render_file with xlsx) or each file on its own sheet (document_merge with ' + 'kind workbook). When the user asks to merge or combine several spreadsheets, wants Excel or ' + 'names no format, and has not said which, ask instead of assuming: return an elicitation whose ' + 'only field is a required single-choice string enum of exactly "' + MERGE_LAYOUT_ONE_SHEET + + '" and "' + MERGE_LAYOUT_SHEET_PER_FILE + '". Do not ask when the request already decides it: ' + 'CSV holds one table, so CSV output means one sheet; one sheet, one table, one list, stack, ' + 'append or combine the rows mean one sheet; a sheet or tab per file, separate sheets or keep them ' + 'separate mean a sheet each; a saved preference, a clarification or an earlier answer also ' + 'decides it. When the user declines to answer, put every row on one sheet and say so in ' + 'assumptions.' +) # The closed set of reasons a deliverable can be unavailable. Each one is a condition the # server can check; the text is application-owned and is what users see. @@ -331,6 +353,33 @@ def build_deliverable_availability(settings, *, capabilities, unavailable=None, 'with pptx and prepared_slide_deck_v1.' )}, ]) + if CAPABILITY_TABULAR_MERGE in available: + recipes.append({'for': 'One file merged from several CSV or Excel files', 'steps': ( + 'tabular_merge with the files in merge order, then render_file binding its "records" ' + 'output with csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1 ' + '(options.sheet_name is optional). Never compose merged rows.' + )}) + if CAPABILITY_TABULAR_INSPECT in available and compose: + recipes.append({'for': 'One file merged from CSV or Excel files whose columns differ', 'steps': ( + 'tabular_inspect with the files; compose binding its "inspection" output and declaring ' + 'a structured-v1 output with profile ' + TABULAR_COLUMN_MAPPING_PROFILE + '; ' + 'tabular_merge with the same files and its "mapping" input bound to that output; then ' + 'render_file as above. When the user wants to review the differences first, compose a ' + 'markdown-v1 answer from the inspection instead and merge in a later turn.' + )}) + facts.append( + 'tabular_merge appends rows exactly; it does not match rows on a key column or change values.' + ) + if CAPABILITY_DOCUMENT_MERGE in available: + recipes.append({'for': 'One PDF, Word document, PowerPoint deck or workbook merged from several files', 'steps': ( + 'document_merge with the files in order and its kind, then render_file binding its ' + '"assembly" output with profile assembled_document_v1 and output_format pdf, docx, pptx, or ' + 'xlsx for a workbook. Never compose a merged document.' + )}) + facts.append('document_merge copies files exactly; it never summarizes, translates or rewrites them.') + if CAPABILITY_TABULAR_MERGE in available and CAPABILITY_DOCUMENT_MERGE in available: + # Only when the plan could deliver either layout is it worth asking which one. + facts.append(MERGE_LAYOUT_FACT) if explicit_images['status'] == 'available': facts.extend([ 'generate_image follows the visual style the user asks for, including photorealistic ' diff --git a/application/single_app/functions_orchestration_document_merge.py b/application/single_app/functions_orchestration_document_merge.py new file mode 100644 index 000000000..978f28515 --- /dev/null +++ b/application/single_app/functions_orchestration_document_merge.py @@ -0,0 +1,183 @@ +# functions_orchestration_document_merge.py +"""Deterministic document merging for Gather / Reason / Render orchestration. + +Version: 0.261.245 +Implemented in: 0.261.245 + +``document_merge`` is a Reason capability. Its adapter in ``functions_orchestration_adapters`` +authorizes the sources; this module reads each original file through the screening-aware +byte reader, assembles them once with ``functions_document_merge`` to check them and to +measure the result, and retains a description of the merge and its report. It creates no +file and calls no model: ``render_file`` assembles the same files again with the +``assembled_document_v1`` profile and delivers the file only when it is byte-identical. +""" + +from copy import deepcopy +import hashlib +import os + +from functions_analysis_access import analysis_source_snapshot +from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_CHAT, + DOCUMENT_ACTION_TYPE_MERGE, + get_document_action_max_documents, +) +from functions_document_merge import ( + DOCUMENT_MERGE_SOURCE_EXTENSIONS, + DocumentMergeError, + DocumentMergeLimits, + MERGE_KIND_DOCX, + MERGE_KIND_PDF, + MERGE_KIND_PPTX, + MERGE_KIND_WORKBOOK, +) +from functions_document_merge_assembly import build_document_assembly, document_merge_options_from_arguments +from functions_mixed_source_orchestration import AUTHORIZATION_STATUS_AUTHORIZED +from functions_orchestration_result_contracts import Completeness, Coverage, ResultContractError +from functions_orchestration_results import NamedOutput + + +DOCUMENT_MERGE_CAPABILITY_ID = "document_merge" +DOCUMENT_MERGE_KINDS = (MERGE_KIND_PDF, MERGE_KIND_DOCX, MERGE_KIND_PPTX, MERGE_KIND_WORKBOOK) +DOCUMENT_MERGE_CHECKS = ( + "authorized_source_snapshots", + "deterministic_assembly", + "assembled_output_digest", +) +# Render's own limit for a chat file, so a merge it checked is one Render can deliver. +_MAX_OUTPUT_MEGABYTES = 500 +_MIB = 1024 * 1024 +_KIND_RESULTS = { + MERGE_KIND_PDF: ("one PDF", ("pages", "page")), + MERGE_KIND_DOCX: ("one Word document", None), + MERGE_KIND_PPTX: ("one PowerPoint deck", ("slides", "slide")), + MERGE_KIND_WORKBOOK: ("one Excel workbook", ("sheets", "sheet")), +} +# Engine codes become application-owned failure codes; their text never reaches a user. +DOCUMENT_MERGE_FAILURE_CODES = { + "unsupported_format": "document_merge_sources_invalid", + "too_few_sources": "document_merge_sources_invalid", + "duplicate_source": "document_merge_sources_invalid", + "invalid_source": "document_merge_sources_invalid", + "encrypted_document": "document_merge_source_unreadable", + "unreadable_document": "document_merge_source_unreadable", + "empty_source": "document_merge_source_unreadable", + "source_unavailable": "document_merge_source_unreadable", + "active_content": "document_merge_active_content", + "too_many_sources": "document_merge_limit_exceeded", + "source_too_large": "document_merge_limit_exceeded", + "input_too_large": "document_merge_limit_exceeded", + "output_too_large": "document_merge_limit_exceeded", + "page_limit_exceeded": "document_merge_limit_exceeded", + "slide_limit_exceeded": "document_merge_limit_exceeded", + "sheet_limit_exceeded": "document_merge_limit_exceeded", + "row_limit_exceeded": "document_merge_limit_exceeded", + "cell_limit_exceeded": "document_merge_limit_exceeded", + "cell_too_large": "document_merge_limit_exceeded", + "sheet_not_found": "merge_sheet_not_found", + "invalid_options": "merge_options_invalid", + "range_invalid": "merge_options_invalid", + "merge_failed": "document_merge_failed", +} + + +def document_merge_failure_code(error): + """The application-owned failure code for an engine error, or the generic step failure.""" + return DOCUMENT_MERGE_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") + + +def document_merge_max_output_bytes(settings): + """The largest merged file chat delivers, the same limit Render applies.""" + try: + megabytes = int((settings or {}).get("max_generated_chat_artifact_size_mb", _MAX_OUTPUT_MEGABYTES)) + except (TypeError, ValueError): + megabytes = _MAX_OUTPUT_MEGABYTES + return max(1, min(megabytes, _MAX_OUTPUT_MEGABYTES)) * _MIB + + +def document_merge_limits(settings): + """The administrator's chat Merge file limit and Render's file size as engine limits.""" + max_parts = get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_CONTEXT_CHAT, settings=settings, + ) + return DocumentMergeLimits( + max_parts=max(2, int(max_parts)), max_output_bytes=document_merge_max_output_bytes(settings), + ) + + +def require_document_merge_manifest(manifest, document_ids, kind): + """The authorized manifest in the planned order, with every file one the kind merges.""" + document_ids = list(document_ids or ()) + if ( + not isinstance(manifest, list) or len(document_ids) < 2 + or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids + or any(source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED for source in manifest) + ): + raise PermissionError("The complete merge source selection is unavailable.") + allowed = DOCUMENT_MERGE_SOURCE_EXTENSIONS[kind] + for source in manifest: + extension = os.path.splitext(str(source.get("file_name") or ""))[1].strip().lower() + if extension not in allowed: + raise DocumentMergeError("unsupported_format", "A selected file isn't the kind of file being merged.") + return manifest + + +def assembled_file_digest(merge_result): + """The SHA-256 of an assembled file, read from its bounded spool.""" + digest = hashlib.sha256() + for chunk in merge_result.iter_chunks(): + digest.update(chunk) + return digest.hexdigest() + + +def persist_document_merge_result( + *, service, producer, kind, options, parts, merge_result, sources, guard_token, input_fingerprint, +): + """Retain the merge description and its report as one complete Reason result.""" + if producer.capability_id != DOCUMENT_MERGE_CAPABILITY_ID: + raise ResultContractError("result_merge_invalid") + snapshots = analysis_source_snapshot(sources) + coverage = Coverage(len(snapshots), len(snapshots), "sources") + report = deepcopy(merge_result.report) + limitations = tuple(report.get("limitations") or ()) + assembly = build_document_assembly(kind, options, parts, merge_result, assembled_file_digest(merge_result)) + outputs = [ + NamedOutput( + "assembly", "structured-v1", assembly, + Completeness("complete", 1, 1, coverage, "valid", DOCUMENT_MERGE_CHECKS, limitations), + ), + NamedOutput( + "report", "structured-v1", report, + Completeness("complete", 1, 1, coverage, "valid", DOCUMENT_MERGE_CHECKS, limitations), + ), + ] + return service.persist_task_result( + producer=producer, role="reason", status="complete", outputs=outputs, + sources=snapshots, origin="grounded", guard_token=guard_token, input_fingerprint=input_fingerprint, + ) + + +def document_merge_step_summary(kind, report): + """A short, application-owned description of a completed document merge.""" + totals = (report or {}).get("totals") or {} + merged = sum(entry.get("status") == "merged" for entry in (report or {}).get("parts") or ()) + label, measure = _KIND_RESULTS[kind] + summary = f"Merged {merged} file(s) into {label}" + if measure: + summary += f" with {totals.get(measure[0], 0):,} {measure[1]}(s)" + return f"{summary}." + + +__all__ = [ + "DOCUMENT_MERGE_CAPABILITY_ID", + "DOCUMENT_MERGE_CHECKS", + "DOCUMENT_MERGE_FAILURE_CODES", + "DOCUMENT_MERGE_KINDS", + "document_merge_failure_code", + "document_merge_limits", + "document_merge_max_output_bytes", + "document_merge_options_from_arguments", + "document_merge_step_summary", + "persist_document_merge_result", + "require_document_merge_manifest", +] diff --git a/application/single_app/functions_orchestration_executor.py b/application/single_app/functions_orchestration_executor.py index 9209833cc..d0a6491f6 100644 --- a/application/single_app/functions_orchestration_executor.py +++ b/application/single_app/functions_orchestration_executor.py @@ -63,9 +63,9 @@ ) from functions_orchestration_deliverables import explicit_image_shortfalls from functions_orchestration_registry import ( - CAPABILITY_TABULAR_ANALYZE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, - DEPENDENCY_PLAN_CONTRACT_VERSION, admitted_export_pairs, external_effect_capability_ids, get_capability, - resolve_available_capability_ids, + CAPABILITY_DOCUMENT_MERGE, CAPABILITY_TABULAR_ANALYZE, CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, + CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, DEPENDENCY_PLAN_CONTRACT_VERSION, + admitted_export_pairs, external_effect_capability_ids, get_capability, resolve_available_capability_ids, ) from functions_orchestration_result_contracts import ( InputBinding, PartialInputNotAcceptedError, ProducerIdentity, ResultContractError, ResultRef, TaskResult, digest, @@ -866,7 +866,10 @@ def _run_dependency_step( readers = resolve_step_inputs(step, context) runtime_step = deepcopy(step) named_sources = [] - if 'sources' in readers and step['capability_id'] == 'document_analyze': + # Analyze, Inspect and Merge accept a search's source set in place of explicit document IDs. + if 'sources' in readers and step['capability_id'] in ( + 'document_analyze', CAPABILITY_TABULAR_INSPECT, CAPABILITY_TABULAR_MERGE, CAPABILITY_DOCUMENT_MERGE, + ): named_sources = read_complete_input(readers['sources']) runtime_step['arguments']['document_ids'] = [ source['document_id'] for source in named_sources diff --git a/application/single_app/functions_orchestration_merge.py b/application/single_app/functions_orchestration_merge.py new file mode 100644 index 000000000..1cd5e5ade --- /dev/null +++ b/application/single_app/functions_orchestration_merge.py @@ -0,0 +1,285 @@ +# functions_orchestration_merge.py +"""Deterministic spreadsheet merging and inspection for Gather / Reason / Render orchestration. + +Version: 0.261.242 +Implemented in: 0.261.239 +Inspection, reconciliation policies and prepared column mappings added in: 0.261.240 + +``tabular_merge`` is a Reason capability and ``tabular_inspect`` a Gather capability. Their +adapters in ``functions_orchestration_adapters`` authorize the sources. This module reads +each original file through the screening-aware byte reader, merges or inspects the files +with ``functions_tabular_merge``, and retains the results. It creates no file and calls no +model: ``render_file`` delivers a file from the retained ``records``, and a compose step +prepares any column mapping. +""" + +from copy import deepcopy + +from content_screening.access import read_available_document_bytes +from functions_analysis_access import analysis_source_snapshot +from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_CHAT, + DOCUMENT_ACTION_TYPE_MERGE, + get_document_action_max_documents, + get_document_action_max_rows, +) +from functions_mixed_source_orchestration import AUTHORIZATION_STATUS_AUTHORIZED, SOURCE_KIND_TABULAR +from functions_orchestration_result_contracts import Completeness, Coverage, RecordColumn, ResultContractError +from functions_orchestration_results import NamedOutput +from functions_tabular_merge import ( + TABULAR_COLUMN_MAPPING_PROFILE, + TabularMergeError, + TabularMergeLimits, + TabularMergeSource, + tabular_inspect_options_from_arguments, + tabular_merge_format_for_file_name, + tabular_merge_options_from_arguments, +) + + +TABULAR_MERGE_CAPABILITY_ID = "tabular_merge" +TABULAR_INSPECT_CAPABILITY_ID = "tabular_inspect" +TABULAR_MERGE_SOURCE_PURPOSE = "native" +TABULAR_MERGE_CHECKS = ( + "authorized_source_snapshots", + "deterministic_row_append", + "exact_row_count", + "schema_policy_enforced", +) +TABULAR_INSPECT_CHECKS = ( + "authorized_source_snapshots", + "bounded_inspection", +) +# Engine codes become application-owned failure codes; their text never reaches a user. +MERGE_FAILURE_CODES = { + "schema_mismatch": "merge_schema_mismatch", + "nothing_to_merge": "merge_nothing_to_merge", + "duplicate_columns": "merge_columns_invalid", + "header_too_long": "merge_columns_invalid", + "too_many_columns": "merge_columns_invalid", + "empty_source": "merge_columns_invalid", + "dedupe_column_not_found": "merge_columns_not_found", + "sort_column_not_found": "merge_columns_not_found", + "row_has_extra_values": "merge_rows_invalid", + "cell_too_large": "merge_rows_invalid", + "unreadable_csv": "merge_source_unreadable", + "unreadable_workbook": "merge_source_unreadable", + "encrypted_workbook": "merge_source_unreadable", + "source_unavailable": "merge_source_unreadable", + "sheet_not_found": "merge_sheet_not_found", + "too_few_sources": "merge_sources_invalid", + "duplicate_source": "merge_sources_invalid", + "unsupported_format": "merge_sources_invalid", + "invalid_source": "merge_sources_invalid", + "invalid_options": "merge_options_invalid", + "invalid_mapping": "merge_mapping_invalid", + "too_many_sources": "merge_limit_exceeded", + "row_limit_exceeded": "merge_limit_exceeded", + "sort_limit_exceeded": "merge_limit_exceeded", + "too_many_sheets": "merge_limit_exceeded", + "source_too_large": "merge_limit_exceeded", + "workbook_too_large": "merge_limit_exceeded", +} +# Inspection reports unreadable files inside its result; only these refuse the whole step. +INSPECT_FAILURE_CODES = { + "too_few_sources": "inspect_sources_invalid", + "duplicate_source": "inspect_sources_invalid", + "unsupported_format": "inspect_sources_invalid", + "invalid_source": "inspect_sources_invalid", + "invalid_options": "inspect_sources_invalid", + "too_many_sources": "inspect_limit_exceeded", + "inspection_too_large": "inspect_limit_exceeded", +} + + +def merge_failure_code(error): + """The application-owned failure code for an engine error, or the generic step failure.""" + return MERGE_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") + + +def inspect_failure_code(error): + """The application-owned failure code for an inspection error, or the generic step failure.""" + return INSPECT_FAILURE_CODES.get(getattr(error, "code", None), "step_failed") + + +def tabular_merge_limits(settings, *, execution_context=DOCUMENT_ACTION_CONTEXT_CHAT): + """The administrator's Merge limits for chat or workflows as engine limits.""" + max_sources = get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings, + ) + max_rows = get_document_action_max_rows(DOCUMENT_ACTION_TYPE_MERGE, execution_context, settings=settings) + return TabularMergeLimits( + max_sources=max(2, int(max_sources)), max_total_rows=int(max_rows), + max_sort_rows=min(int(max_rows), TabularMergeLimits.max_sort_rows), + ) + + +def tabular_merge_options(arguments, mapping=None): + """Engine options from validated step arguments and an optional validated mapping.""" + return tabular_merge_options_from_arguments(arguments, mapping=mapping) + + +def tabular_inspect_options(arguments): + """Inspection options from validated step arguments.""" + return tabular_inspect_options_from_arguments(arguments) + + +def require_tabular_merge_manifest(manifest, document_ids, *, minimum=2): + """The authorized manifest in the planned order, with every source a CSV or Excel file.""" + document_ids = list(document_ids or ()) + if ( + not isinstance(manifest, list) or len(document_ids) < minimum + or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids + or any(source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED for source in manifest) + ): + raise PermissionError("The complete merge source selection is unavailable.") + for source in manifest: + if ( + source.get("source_kind") != SOURCE_KIND_TABULAR + or tabular_merge_format_for_file_name(source.get("file_name")) is None + ): + raise TabularMergeError("unsupported_format", "Only CSV and Excel files can be merged.") + return manifest + + +def build_tabular_merge_sources(manifest, user_id, *, byte_reader=None): + """Engine sources whose bytes are read, one file at a time, through the access boundary.""" + reader = byte_reader or read_available_document_bytes + sources = [] + for entry in manifest: + source = deepcopy(entry) + + def load(source=source): + _document, content = reader( + source, user_id, source.get("group_id"), source.get("public_workspace_id"), + purpose=TABULAR_MERGE_SOURCE_PURPOSE, + ) + return content + + sources.append(TabularMergeSource( + source_id=source["document_id"], + file_name=source.get("file_name") or source.get("display_name") or source["document_id"], + load_bytes=load, + )) + return sources + + +def build_document_merge_parts(manifest, user_id, *, byte_reader=None): + """Document merge parts whose bytes are read, one file at a time, through the access boundary.""" + # The document merge engine loads only when files are assembled. + from functions_document_merge import DocumentMergePart + + reader = byte_reader or read_available_document_bytes + parts = [] + for entry in manifest: + source = deepcopy(entry) + + def load(source=source): + _document, content = reader( + source, user_id, source.get("group_id"), source.get("public_workspace_id"), + purpose=TABULAR_MERGE_SOURCE_PURPOSE, + ) + return content + + parts.append(DocumentMergePart( + source_id=source["document_id"], + file_name=source.get("file_name") or source.get("display_name") or source["document_id"], + load_bytes=load, + )) + return parts + + +def merge_report_with_mapping(report, mapping): + """The merge report, recording what a prepared column mapping asked for.""" + if mapping is not None: + report["mapping"] = { + "profile": TABULAR_COLUMN_MAPPING_PROFILE, + "explicitly_ignored": list(mapping["ignored"]), + "low_confidence": deepcopy(mapping["low_confidence"]), + "notes": mapping["notes"], + } + return report + + +def persist_tabular_merge_result( + *, service, producer, merge_result, sources, guard_token, input_fingerprint, mapping=None, +): + """Retain the merged rows and the merge report as one complete Reason result.""" + if producer.capability_id != TABULAR_MERGE_CAPABILITY_ID: + raise ResultContractError("result_merge_invalid") + snapshots = analysis_source_snapshot(sources) + nullable = merge_result.nullable_columns + columns = tuple(RecordColumn(name, "string", name in nullable) for name in merge_result.columns) + row_count = merge_result.row_count + coverage = Coverage(len(snapshots), len(snapshots), "sources") + report = merge_report_with_mapping(merge_result.report, mapping) + limitations = tuple(report.get("limitations") or ()) + outputs = [ + NamedOutput( + "records", "records-v1", merge_result.iter_records(), + Completeness("complete", row_count, row_count, coverage, "valid", TABULAR_MERGE_CHECKS, limitations), + columns, + ), + NamedOutput( + "report", "structured-v1", report, + Completeness("complete", 1, 1, coverage, "valid", TABULAR_MERGE_CHECKS, limitations), + ), + ] + return service.persist_task_result( + producer=producer, role="reason", status="complete", outputs=outputs, + sources=snapshots, origin="grounded", guard_token=guard_token, input_fingerprint=input_fingerprint, + ) + + +def persist_tabular_inspection_result(*, service, producer, inspection, sources, guard_token, input_fingerprint): + """Retain the inspection as one complete Gather result.""" + if producer.capability_id != TABULAR_INSPECT_CAPABILITY_ID: + raise ResultContractError("result_inspection_invalid") + snapshots = analysis_source_snapshot(sources) + coverage = Coverage(len(snapshots), len(snapshots), "sources") + limitations = list(inspection.get("limitations") or ()) + problems = sum(entry.get("status") == "problem" for entry in inspection.get("sources") or ()) + if problems: + limitations.append(f"{problems} file(s) couldn't be read; the inspection reports why.") + outputs = [ + NamedOutput( + "inspection", "structured-v1", inspection, + Completeness("complete", 1, 1, coverage, "valid", TABULAR_INSPECT_CHECKS, tuple(limitations)), + ), + ] + return service.persist_task_result( + producer=producer, role="gather", status="complete", outputs=outputs, + sources=snapshots, origin="grounded", guard_token=guard_token, input_fingerprint=input_fingerprint, + ) + + +def merge_step_summary(merge_result): + """A short, application-owned description of a completed merge.""" + totals = (merge_result.report or {}).get("totals") or {} + summary = ( + f"Merged {totals.get('rows', merge_result.row_count):,} row(s) from " + f"{totals.get('sources', 0)} file(s) into {totals.get('columns', len(merge_result.columns))} column(s)" + ) + details = [] + if totals.get("duplicates_removed"): + details.append(f"removed {totals['duplicates_removed']:,} duplicate row(s)") + if totals.get("excluded"): + details.append(f"left out {totals['excluded']} file(s) or sheet(s) whose columns don't fit") + return f"{summary}; {'; '.join(details)}." if details else f"{summary}." + + +def inspection_step_summary(inspection): + """A short, application-owned description of a completed inspection.""" + sources = inspection.get("sources") or [] + compatibility = inspection.get("compatibility") or {} + problems = sum(entry.get("status") == "problem" for entry in sources) + if compatibility.get("identical_columns"): + fit = "they all have the same columns" + elif compatibility.get("tables"): + fit = "their columns differ" + else: + fit = "no columns could be read" + summary = f"Inspected {len(sources)} file(s); {fit}" + if problems: + summary += f"; {problems} couldn't be read" + return f"{summary}." diff --git a/application/single_app/functions_orchestration_planner.py b/application/single_app/functions_orchestration_planner.py index ff456399f..1372c6c0a 100644 --- a/application/single_app/functions_orchestration_planner.py +++ b/application/single_app/functions_orchestration_planner.py @@ -285,6 +285,26 @@ def resolve_planner_client(settings): words such as "this week" can stay relative. - optionally "alerts" {"mode":"every_run"|"failures_only","severity":"info"|"low"}, "run_as" "self"|"none", and "durable": true. +- A task can merge files with code instead of a model by adding "merge": {"kind","files", + "output_format","file_name","options"} and no agent runner. kind "tabular" (the default) appends + the rows of CSV and Excel files into one "csv" or "xlsx" file; "workbook" puts each CSV or Excel + file on its own sheet of one "xlsx" workbook; "pdf" joins PDFs into one "pdf"; "docx" appends + Word documents into one "docx"; "pptx" appends the slides of PowerPoint decks into one "pptx". + When the user wants spreadsheets merged into one Excel file, or names no format, and has not + said whether every row goes on one sheet (kind "tabular" with "xlsx") or each file keeps its + own sheet (kind "workbook"), ask before proposing: one required single-choice string enum of + exactly "All rows on one sheet" and "Each file on its own sheet". "csv" output always means + one sheet. + files "inputs" + merges the task's two or more input documents in order; "changed" merges the files a file_sync + trigger added or changed; "all" merges every matching file in the user's personal workspace; + "recent" merges those added or changed in the last "recent_window_minutes". For tabular, + "options" takes tabular_merge's column, sheet, duplicate and sort settings, with + "column_aliases" written as [{"column","aliases":[...]}]; workbook takes "sheets" and "sheet"; + pdf takes "bookmarks"; docx takes "formatting", "page_breaks" and "source_headings"; pptx takes + "formatting" and "sections". Propose one when the user wants files merged on a schedule or on + every sync, or more files than one chat merge allows; a one-time merge of a few files runs in + chat with tabular_merge or document_merge. It needs no task_actions. Refer to agents, documents and File Sync sources only by their workflow_planning.catalog handles, and never write a record id, model or endpoint into a blueprint. "task_actions" lists, for each task in order, the action kinds it needs from the capability's input schema, or [] when it needs none. @@ -421,7 +441,11 @@ def resolve_planner_client(settings): invent one. If the user already selected documents, plan around those documents. Use each candidate's server-resolved source_kind, not its display label, to choose compatible work. Never send tabular source IDs to document_analyze or document_compare: those steps -do not admit native tabular inputs. For mixed narrative/tabular comparisons, prepare each +do not admit native tabular inputs. To combine the rows of several tabular sources into one +table or file, use tabular_merge with every one of those sources, never compose; it appends +rows and cannot match rows on a key, so ask when the user may mean that. When their columns +may differ, choose union, aliases or exclusion, or inspect them with tabular_inspect first. +For mixed narrative/tabular comparisons, prepare each source with compatible offered capabilities and compose their named results. Ask each preparation step only for its own sources' contribution, such as values, periods, units and identifiers. A preparation step sees only its own sources, so never ask it to compare them @@ -577,8 +601,10 @@ def resolve_planner_client(settings): user's request, without replacing the original "message" or user selections. Plan around accepted source identities and explanations, including on repeated questions. Do not repeat a question that clarifications or earlier runs already answered or declined. Only ask when you -truly cannot proceed; a reasonable assumption about what the user means, stated in -"assumptions", is better than a question. +truly cannot proceed, or when a capability's when_to_use, a fact in +capability_availability.deliverables or the workflow instructions say to ask; otherwise a +reasonable assumption about what the user means, stated in "assumptions", is better than a +question. """ def build_planner_messages( diff --git a/application/single_app/functions_orchestration_registry.py b/application/single_app/functions_orchestration_registry.py index b31516ca8..30575fe22 100644 --- a/application/single_app/functions_orchestration_registry.py +++ b/application/single_app/functions_orchestration_registry.py @@ -43,6 +43,8 @@ from functions_orchestration_result_contracts import ( IMAGE_ASSET_KIND, RESULT_KINDS, ResultContractError, canonical_bytes, ) +# The merge engine imports only the standard library, so the registry stays importable alone. +from functions_tabular_merge import TABULAR_COLUMN_MAPPING_PROFILE # The schema marker every saved plan carries. A plan without it, or with the earlier value, # was written by the removed legacy contract and is never opened or run. @@ -77,6 +79,7 @@ def effective_capability_ids(values): # produced the `_load_orchestration_helper` shim in `functions_tabular_analysis.py`. DOCUMENT_ACTION_TYPE_ANALYZE = 'analyze' DOCUMENT_ACTION_TYPE_COMPARISON = 'comparison' +DOCUMENT_ACTION_TYPE_MERGE = 'merge' DOCUMENT_ACTION_CONTEXT_CHAT = 'chat' COST_CLASS_LOW = 'low' @@ -96,6 +99,9 @@ def effective_capability_ids(values): CAPABILITY_DOCUMENT_ANALYZE = 'document_analyze' CAPABILITY_DOCUMENT_COMPARE = 'document_compare' CAPABILITY_TABULAR_ANALYZE = 'tabular_analyze' +CAPABILITY_TABULAR_INSPECT = 'tabular_inspect' +CAPABILITY_TABULAR_MERGE = 'tabular_merge' +CAPABILITY_DOCUMENT_MERGE = 'document_merge' CAPABILITY_WEB_SEARCH = 'web_search' CAPABILITY_URL_FETCH = 'url_fetch' CAPABILITY_DEEP_RESEARCH = 'deep_research' @@ -572,6 +578,27 @@ def generate_image_arguments_schema(options=None): ' This step reads narrative documents; native tabular handoff is not admitted.' ) +# Argument shapes shared by the spreadsheet merge and inspection steps. Descriptors are +# deep-copied whenever they are read, so sharing them here never shares a caller's copy. +_MERGE_COLUMN_NAME_INPUT = {'type': 'string', 'minLength': 1, 'maxLength': 256} +_MERGE_SHEET_INPUT = { + 'type': 'string', + 'minLength': 1, + 'maxLength': 31, + 'description': "One sheet to read in every Excel file. Omit for each workbook's first visible sheet.", +} +_MERGE_SHEETS_INPUT = { + 'type': 'string', + 'enum': ['first', 'all'], + 'description': 'all reads every visible sheet of each workbook; first (default) reads one.', +} +_MERGE_HEADER_ROW_INPUT = { + 'type': 'integer', + 'minimum': 1, + 'maximum': 1000, + 'description': 'The row holding the column headers in every file. Omit to use the first nonblank row.', +} + # The registry itself, in the order a plan tends to read: gather, then reason, then render. # @@ -779,6 +806,326 @@ def generate_image_arguments_schema(options=None): 'max_per_plan': 2, 'adapter': CAPABILITY_TABULAR_ANALYZE, }, + { + 'id': CAPABILITY_TABULAR_INSPECT, + 'label': 'Inspect spreadsheets', + 'role': ROLE_GATHER, + 'request_gate': None, + 'summary': ( + 'Read the sheets, column headers, row counts and a few sample rows of CSV or Excel files, ' + 'and how their columns line up.' + ), + 'when_to_use': ( + 'Use before tabular_merge when the files may not share the same columns, or when the user ' + 'asks what sheets, columns or rows CSV or Excel files have. It reports each file\'s headers, ' + 'row counts and sample rows, which files share the same columns, near matches such as ' + '"Customer ID" and "customer_id", a likely title row with the header_row to use, and a ' + 'suggested schema_policy. A file that cannot be read is reported, not fatal. To line up ' + 'differently named columns, bind its "inspection" output to a compose step that declares ' + 'a structured-v1 output with profile ' + TABULAR_COLUMN_MAPPING_PROFILE + ', and bind ' + 'that output as tabular_merge\'s "mapping" input. When the user should review the ' + 'differences first, compose an answer from the inspection and merge in a later turn.' + ), + 'settings_gates': (), + 'settings_gates_any': ( + 'enable_user_workspace', + 'enable_group_workspaces', + 'enable_public_workspaces', + ), + 'gate': _document_action_gate(DOCUMENT_ACTION_TYPE_MERGE), + 'requires_scope': (SCOPE_PERSONAL, SCOPE_GROUP, SCOPE_PUBLIC), + 'inputs': { + 'type': 'object', + 'properties': { + 'document_ids': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 1, + 'uniqueItems': True, + 'description': 'The CSV or Excel files to inspect.', + }, + 'doc_scope': { + 'type': 'string', + 'enum': ['all', 'personal', 'group', 'public'], + 'default': 'all', + }, + 'sheet': _MERGE_SHEET_INPUT, + 'sheets': _MERGE_SHEETS_INPUT, + 'header_row': _MERGE_HEADER_ROW_INPUT, + 'sample_rows': { + 'type': 'integer', + 'minimum': 0, + 'maximum': 10, + 'description': 'Sample rows to read per sheet; 3 when omitted.', + }, + }, + 'additionalProperties': False, + }, + 'result_contract_version': 'tabular-inspect-v1', + 'result_outputs': {'inspection': 'structured-v1'}, + # Files a search found can be inspected by binding its source set by name. + 'result_input_kinds': {'sources': ('source-set-v1',)}, + 'partial_inputs_supported': False, + 'produces': (PRODUCES_RETAINED_RESULTS,), + 'cost_class': COST_CLASS_LOW, + 'max_per_plan': 2, + 'adapter': CAPABILITY_TABULAR_INSPECT, + # Inspection serves Merge, so Merge's switch and chat file limit govern it too. + 'document_action_type': DOCUMENT_ACTION_TYPE_MERGE, + }, + { + 'id': CAPABILITY_TABULAR_MERGE, + 'label': 'Merge spreadsheets', + 'role': ROLE_REASON, + 'request_gate': None, + 'summary': ( + 'Combine the rows of two or more CSV or Excel files into one table, exactly and without ' + 'a model, lining up columns by name, alias or a prepared mapping.' + ), + 'when_to_use': ( + 'Use when the user asks to merge, combine, append, stack or consolidate several CSV or ' + 'Excel files into one table or file. Every row is kept, in the order the files are listed. ' + 'It never matches rows on a key column (no joins or lookups) and never edits values; when ' + '"merge" could mean matching rows on a key, ask. Name the files in document_ids in merge ' + "order, or bind inputs.sources to a search step's sources output when the files must be " + 'found first. schema_policy: by_name (default) needs the same columns in any order, ' + 'exact_order also the same order, union keeps every column from every file and leaves a ' + 'missing one blank, mapped keeps exactly "columns". column_aliases lines up other header ' + 'names, e.g. {"Customer ID": ["cust_id"]}. When the columns may differ and how they line ' + 'up is unclear, run tabular_inspect first and bind a compose step\'s ' + + TABULAR_COLUMN_MAPPING_PROFILE + ' output as the "mapping" input, without columns or ' + 'aliases. on_incompatible exclude leaves out, and reports, files or sheets that do not fit ' + 'instead of failing. header_row skips title rows above the headers. sheet picks one sheet; ' + 'sheets all merges every visible sheet and adds a Source Sheet column. dedupe removes ' + 'duplicate rows: exact_rows, or key_columns with dedupe_columns, keeping the first or last ' + '(dedupe_keep). sort_by orders rows by up to three columns. include_source_column adds ' + 'each row\'s file name. To deliver a file, bind the records output to render_file with ' + 'csv and exact_tabular_records_v1, or xlsx and exact_tabular_workbook_v1. Never compose ' + 'merged rows. A merge of more files than the chat limit, or one that should repeat, ' + 'belongs in a workflow.' + ), + 'settings_gates': (), + 'settings_gates_any': ( + 'enable_user_workspace', + 'enable_group_workspaces', + 'enable_public_workspaces', + ), + 'gate': _document_action_gate(DOCUMENT_ACTION_TYPE_MERGE), + 'requires_scope': (SCOPE_PERSONAL, SCOPE_GROUP, SCOPE_PUBLIC), + 'inputs': { + 'type': 'object', + 'properties': { + 'document_ids': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 2, + 'uniqueItems': True, + 'description': 'The CSV or Excel files to merge, in the order their rows should appear.', + }, + 'doc_scope': { + 'type': 'string', + 'enum': ['all', 'personal', 'group', 'public'], + 'default': 'all', + }, + 'schema_policy': { + 'type': 'string', + 'enum': ['by_name', 'exact_order', 'union', 'mapped'], + 'default': 'by_name', + 'description': ( + 'by_name: same columns in any order. exact_order: same columns in the same ' + 'order. union: every column from every file. mapped: exactly "columns".' + ), + }, + 'columns': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 255, + 'uniqueItems': True, + 'description': 'mapped only: the merged columns, in order.', + }, + 'column_aliases': { + 'type': 'object', + 'maxProperties': 256, + 'additionalProperties': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 32, + 'uniqueItems': True, + }, + 'description': 'Each column name mapped to the other header names that mean it.', + }, + 'sheet': _MERGE_SHEET_INPUT, + 'sheets': _MERGE_SHEETS_INPUT, + 'header_row': _MERGE_HEADER_ROW_INPUT, + 'include_source_column': { + 'type': 'boolean', + 'default': True, + 'description': "Add a column holding each row's file name.", + }, + 'source_column_name': { + 'type': 'string', + 'minLength': 1, + 'maxLength': 128, + 'default': 'Source File', + }, + 'on_incompatible': { + 'type': 'string', + 'enum': ['fail', 'exclude'], + 'description': 'fail (default) merges nothing when a file does not fit; exclude leaves it out.', + }, + 'dedupe': { + 'type': 'string', + 'enum': ['none', 'exact_rows', 'key_columns'], + 'description': 'Remove duplicate rows; none when omitted.', + }, + 'dedupe_columns': { + 'type': 'array', + 'items': _MERGE_COLUMN_NAME_INPUT, + 'minItems': 1, + 'maxItems': 16, + 'uniqueItems': True, + 'description': 'key_columns only: the columns that identify a duplicate.', + }, + 'dedupe_keep': { + 'type': 'string', + 'enum': ['first', 'last'], + 'description': 'Which duplicate to keep; first when omitted.', + }, + 'sort_by': { + 'type': 'array', + 'maxItems': 3, + 'items': { + 'type': 'object', + 'properties': { + 'column': _MERGE_COLUMN_NAME_INPUT, + 'descending': {'type': 'boolean'}, + 'value_type': {'type': 'string', 'enum': ['text', 'number', 'date']}, + }, + 'required': ['column'], + 'additionalProperties': False, + }, + 'description': 'Sort keys, most significant first; text ascending when not stated.', + }, + }, + 'additionalProperties': False, + }, + 'result_contract_version': 'tabular-merge-v1', + 'result_outputs': {'records': 'records-v1', 'report': 'structured-v1'}, + # Files a search found can be merged by binding its source set by name, and a compose + # step's prepared column mapping lines up differently named columns. + 'result_input_kinds': {'sources': ('source-set-v1',), 'mapping': ('structured-v1',)}, + 'partial_inputs_supported': False, + 'produces': (PRODUCES_RETAINED_RESULTS,), + 'cost_class': COST_CLASS_MEDIUM, + 'max_per_plan': 2, + 'adapter': CAPABILITY_TABULAR_MERGE, + # Enforced by the validator against the administrator's chat limit for Merge. + 'document_action_type': DOCUMENT_ACTION_TYPE_MERGE, + }, + { + 'id': CAPABILITY_DOCUMENT_MERGE, + 'label': 'Merge documents', + 'role': ROLE_REASON, + 'request_gate': None, + 'summary': ( + 'Combine two or more PDFs, Word documents or PowerPoint decks into one file of the same ' + 'kind, or CSV and Excel files into one workbook with a sheet per file, exactly and ' + 'without a model.' + ), + 'when_to_use': ( + 'Use when the user asks to combine, join, append, put together or bind several PDFs, Word ' + 'documents or PowerPoint decks into one file, or to keep several CSV or Excel files as ' + 'separate sheets of one workbook. Files are copied exactly, in the order listed; nothing ' + 'is summarized or rewritten. Name the files in document_ids in merge order, or bind ' + "inputs.sources to a search step's sources output when the files must be found first. " + 'kind is required: pdf for PDFs (bookmarks false leaves out the bookmark per file); docx ' + 'for Word documents (formatting keep_source or use_first, page_breaks, source_headings); ' + 'pptx for PowerPoint decks (formatting, sections); workbook for CSV or Excel files, one ' + 'sheet per file (sheets all copies every visible sheet, or sheet names one). Every file ' + 'must be the kind being merged. To deliver the file, bind the assembly output to ' + 'render_file with profile assembled_document_v1 and output_format pdf, docx, pptx, or ' + 'xlsx for workbook. To append the rows of spreadsheets into one table, use tabular_merge ' + 'instead. A merge of more files than the chat limit, or one that should repeat, belongs ' + 'in a workflow.' + ), + 'settings_gates': (), + 'settings_gates_any': ( + 'enable_user_workspace', + 'enable_group_workspaces', + 'enable_public_workspaces', + ), + 'gate': _document_action_gate(DOCUMENT_ACTION_TYPE_MERGE), + 'requires_scope': (SCOPE_PERSONAL, SCOPE_GROUP, SCOPE_PUBLIC), + 'inputs': { + 'type': 'object', + 'properties': { + 'document_ids': { + 'type': 'array', + 'items': {'type': 'string'}, + 'minItems': 2, + 'uniqueItems': True, + 'description': 'The files to merge, in the order they should appear.', + }, + 'doc_scope': { + 'type': 'string', + 'enum': ['all', 'personal', 'group', 'public'], + 'default': 'all', + }, + 'kind': { + 'type': 'string', + 'enum': ['pdf', 'docx', 'pptx', 'workbook'], + 'description': 'The kind of file to create; every file must be that kind.', + }, + 'bookmarks': { + 'type': 'boolean', + 'description': 'pdf only: a bookmark named after each file; true when omitted.', + }, + 'formatting': { + 'type': 'string', + 'enum': ['keep_source', 'use_first'], + 'description': ( + "docx and pptx only: keep_source (default) keeps each file's look; use_first " + "uses the first file's styles or theme for all of them." + ), + }, + 'page_breaks': { + 'type': 'boolean', + 'description': 'docx only: start each document on a new page; true when omitted.', + }, + 'source_headings': { + 'type': 'boolean', + 'description': 'docx only: put a heading with the file name before each document.', + }, + 'sections': { + 'type': 'boolean', + 'description': "pptx only: group each deck's slides in a section; true when omitted.", + }, + 'sheet': { + **_MERGE_SHEET_INPUT, + 'description': "workbook only: the one sheet to copy from every file.", + }, + 'sheets': { + **_MERGE_SHEETS_INPUT, + 'description': 'workbook only: all copies every visible sheet; first (default) copies one.', + }, + }, + 'required': ['kind'], + 'additionalProperties': False, + }, + 'result_contract_version': 'document-merge-v1', + 'result_outputs': {'assembly': 'structured-v1', 'report': 'structured-v1'}, + # Files a search found can be merged by binding its source set by name. + 'result_input_kinds': {'sources': ('source-set-v1',)}, + 'partial_inputs_supported': False, + 'produces': (PRODUCES_RETAINED_RESULTS,), + 'cost_class': COST_CLASS_MEDIUM, + 'max_per_plan': 2, + 'adapter': CAPABILITY_DOCUMENT_MERGE, + # Enforced by the validator against the administrator's chat limit for Merge. + 'document_action_type': DOCUMENT_ACTION_TYPE_MERGE, + }, { 'id': CAPABILITY_WEB_SEARCH, 'label': 'Search the web', diff --git a/application/single_app/functions_orchestration_rendering.py b/application/single_app/functions_orchestration_rendering.py index df04fde74..e6f3c844f 100644 --- a/application/single_app/functions_orchestration_rendering.py +++ b/application/single_app/functions_orchestration_rendering.py @@ -35,7 +35,7 @@ GeneratedFileExportRequest, GeneratedFileExportStream, ) -from functions_generated_export_registry import resolve_generated_file_export_format +from functions_generated_export_registry import ASSEMBLED_DOCUMENT_PROFILE, resolve_generated_file_export_format from functions_generated_file_exports import build_generated_file_export from functions_orchestration_artifacts import ( OrchestrationArtifactTransport, @@ -181,6 +181,68 @@ def _source_visibility_code(error): return None +def _raise_source_read_failure(exc): + """Raise the output failure a failed source read means; return when ``exc`` should be raised as is. + + Call it from an ``except`` block and re-raise when it returns. The checks run in the order + a source read's failures must be told apart, so infrastructure faults stay retryable. + """ + if isinstance(exc, ScreeningError): + raise_output_read_infrastructure_failure(exc) + return + if isinstance(exc, (OutputError, OutputUnavailableError, OutputStorageError)): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + if output_storage_failure(exc) is not None: + raise_output_read_infrastructure_failure(exc) + return + if isinstance(exc, ResultUnavailableError): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + if exc.code in _SOURCE_CONFIGURATION_CODES: + raise OutputError("output_source_configuration_invalid") from exc + raise OutputUnavailableError(_source_visibility_code(exc)) from exc + if isinstance(exc, PermissionError): + raise OutputUnavailableError(_source_visibility_code(exc)) from exc + if isinstance(exc, LookupError): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + if type(exc) is not LookupError: + return + raise OutputUnavailableError("output_source_unavailable") from exc + if isinstance(exc, (ResourceNotFoundError, CosmosResourceNotFoundError)): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + raise OutputUnavailableError("output_source_unavailable") from exc + if isinstance(exc, AnalysisWorkUnitConflictError): + code = _source_visibility_code(exc) + if code is None: + return + raise OutputUnavailableError(code) from exc + if isinstance(exc, (AzureError, TimeoutError, ConnectionError)): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + raise OutputStorageError() from exc + if isinstance(exc, (ResultContractError, WorkflowResultIntegrityError)): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + if output_storage_failure(exc) is not None: + raise_output_read_infrastructure_failure(exc) + raise OutputError("output_invalid_result") from exc + if isinstance(exc, (RuntimeError, ValueError, TypeError, AttributeError, OSError)): + authority = external_authority_failure(exc) + if authority is not None: + raise authority[0] + if output_storage_failure(exc) is not None: + raise_output_read_infrastructure_failure(exc) + + def output_failure(exc): """Classify once without persisting or exposing provider exception messages.""" if isinstance(exc, OutputStepTimeLimitError): @@ -207,6 +269,9 @@ def output_failure(exc): if isinstance(exc, ResultContractError): return "output_invalid_result", False if isinstance(exc, GeneratedFileExportError): + if exc.code == "source_changed": + # A merged file assembled again from its originals no longer matches the checked bytes. + return "output_source_changed", False code = exc.code if exc.code in _VALIDATION_CODES | _TRANSIENT_RENDER_CODES else "render_validation_failed" return code, bool(exc.retryable is True and code in _TRANSIENT_RENDER_CODES) if isinstance(exc, (TimeoutError, ConnectionError)): @@ -514,8 +579,8 @@ class OrchestrationRenderingService: def __init__( self, store, results, transport, *, authorize_execution, max_output_bytes, limits=None, office_limits=None, image_resolver=None, - image_asset_reader=None, renderer=build_generated_file_export, jitter=random.random, - full_check_interval=RENDER_FULL_CHECK_INTERVAL_SECONDS, monotonic=time.monotonic, + image_asset_reader=None, document_bytes_reader=None, renderer=build_generated_file_export, + jitter=random.random, full_check_interval=RENDER_FULL_CHECK_INTERVAL_SECONDS, monotonic=time.monotonic, ): if ( not isinstance(store, OrchestrationOutputStore) @@ -526,6 +591,7 @@ def __init__( or results.access.user_id != store.user_id or results.access.conversation_id != store.conversation_id or (image_asset_reader is not None and not callable(image_asset_reader)) + or (document_bytes_reader is not None and not callable(document_bytes_reader)) ): raise OutputError("output_service_required") if type(max_output_bytes) is not int or not 1 <= max_output_bytes <= MAX_OUTPUT_BYTES: @@ -549,6 +615,9 @@ def __init__( # image descriptor; the owner supplies it. The service decides which images a file # may contain and verifies every byte it is given. self.image_asset_reader = image_asset_reader + # ``document_bytes_reader(source)`` returns one workspace document's current bytes with + # the actor's access; the owner supplies it. Only a merged file's own sources are read. + self.document_bytes_reader = document_bytes_reader self.renderer = renderer self.jitter = jitter self.full_check_interval = full_check_interval @@ -611,6 +680,51 @@ def resolve(reference): return resolve + def _document_reader_for(self, record): + """Read only the original files a merged file was assembled from, as they are now. + + The merge result's own lineage names them, and any other document is refused. A retained + result takes its access from its conversation and run, but a merged file is assembled from + these files when it renders, so each one is read through the owner's reader with the + actor's current access, and the result's producer chain is checked again after every read. + The renderer delivers the file only if it assembles to the checked bytes. + """ + if self.document_bytes_reader is None or record["render_spec"].get("profile") != ASSEMBLED_DOCUMENT_PROFILE: + return None + source = ResultRef.from_dict(record["source_ref"]) + cache = {} + + def documents(): + if "documents" not in cache: + try: + reader = self.results.open_result(source, require_current_sources=True) + lineage = reader.lineage_sources() + except Exception as exc: + _raise_source_read_failure(exc) + raise + by_id = {item.get("document_id"): item for item in lineage if type(item) is dict} + if len(by_id) != len(lineage): + raise OutputUnavailableError("output_source_unavailable") + cache.update(reader=reader, documents=by_id) + return cache["documents"] + + def read(document_id): + snapshot = documents().get(document_id) if type(document_id) is str else None + if snapshot is None: + raise OutputUnavailableError("output_source_unavailable") + try: + content = self.document_bytes_reader(deepcopy(snapshot)) + cache["reader"].recheck() + except Exception as exc: + # A storage fault while reading a file stays retryable; a revoked or held file doesn't. + _raise_source_read_failure(exc) + raise + if type(content) is not bytes: + raise OutputUnavailableError("output_source_unavailable") + return content + + return read + @staticmethod def _read_metadata(read, *args, **kwargs): """A metadata I/O permission fault is not a source-access decision.""" @@ -633,60 +747,8 @@ def _authorize(self, record, operation): return self.results.open_result( reference, require_current_sources=record["render_spec"]["require_current_sources"], ) - except ScreeningError as exc: - raise_output_read_infrastructure_failure(exc) - raise - except (OutputError, OutputUnavailableError, OutputStorageError) as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - if output_storage_failure(exc) is not None: - raise_output_read_infrastructure_failure(exc) - raise - except ResultUnavailableError as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - if exc.code in _SOURCE_CONFIGURATION_CODES: - raise OutputError("output_source_configuration_invalid") from exc - raise OutputUnavailableError(_source_visibility_code(exc)) from exc - except PermissionError as exc: - raise OutputUnavailableError(_source_visibility_code(exc)) from exc - except LookupError as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - if type(exc) is not LookupError: - raise - raise OutputUnavailableError("output_source_unavailable") from exc - except (ResourceNotFoundError, CosmosResourceNotFoundError) as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - raise OutputUnavailableError("output_source_unavailable") from exc - except AnalysisWorkUnitConflictError as exc: - code = _source_visibility_code(exc) - if code is None: - raise - raise OutputUnavailableError(code) from exc - except (AzureError, TimeoutError, ConnectionError) as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - raise OutputStorageError() from exc - except (ResultContractError, WorkflowResultIntegrityError) as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - if output_storage_failure(exc) is not None: - raise_output_read_infrastructure_failure(exc) - raise OutputError("output_invalid_result") from exc - except (RuntimeError, ValueError, TypeError, AttributeError, OSError) as exc: - authority = external_authority_failure(exc) - if authority is not None: - raise authority[0] - if output_storage_failure(exc) is not None: - raise_output_read_infrastructure_failure(exc) + except Exception as exc: + _raise_source_read_failure(exc) raise def ensure_output( @@ -1029,6 +1091,7 @@ def _render_attempt(self, output_id, claim, worker_id, attempt): max_output_bytes=min(self.max_output_bytes, record["render_spec"]["max_output_bytes"]), check=check, limits=bounded_limits, office_limits=self.office_limits, image_resolver=self._image_resolver_for(record, entry), + **({"document_reader": reader} if (reader := self._document_reader_for(record)) else {}), ) with ClosingExportResource(rendered): with attempt.phase("verify"): diff --git a/application/single_app/functions_orchestration_result_contracts.py b/application/single_app/functions_orchestration_result_contracts.py index cd1674a62..ca2286832 100644 --- a/application/single_app/functions_orchestration_result_contracts.py +++ b/application/single_app/functions_orchestration_result_contracts.py @@ -33,7 +33,9 @@ }) RESULT_STATES = frozenset({"pending", "partial", "complete", "invalid", "cancelled", "failed", "unavailable"}) RESULT_ROLES = frozenset({"gather", "reason", "render"}) -REASON_CAPABILITIES = frozenset({"document_analyze", "document_compare", "tabular_analyze"}) +REASON_CAPABILITIES = frozenset({ + "document_analyze", "document_compare", "tabular_analyze", "tabular_merge", "document_merge", +}) MAX_OUTPUTS = 32 MAX_COLUMNS = 256 MAX_BINDING_STEPS = 64 diff --git a/application/single_app/functions_orchestration_results.py b/application/single_app/functions_orchestration_results.py index ecffb062b..bd935e4ed 100644 --- a/application/single_app/functions_orchestration_results.py +++ b/application/single_app/functions_orchestration_results.py @@ -762,6 +762,14 @@ def upstream_references(self): self.recheck() return tuple(ResultRef.from_dict(item) for item in self._manifest["lineage"]["upstream"]) + def lineage_sources(self): + """The workspace documents this result was prepared from, after current access checks. + + Only these may be read again for it; a merged file is assembled from exactly them. + """ + self.recheck() + return deepcopy(self._manifest["lineage"]["sources"]) + def _collection(self): output = self._manifest["outputs"][self.reference.output_name] return _collection_manifest(self.reference.producer, self.reference.output_name, output["storage"]) diff --git a/application/single_app/functions_orchestration_schema.py b/application/single_app/functions_orchestration_schema.py index deaab61d1..478c5b4ad 100644 --- a/application/single_app/functions_orchestration_schema.py +++ b/application/single_app/functions_orchestration_schema.py @@ -56,8 +56,11 @@ APPROVAL_FLOOR_MANUAL, CAPABILITY_ACTION_INVOKE, CAPABILITY_COMPOSE, + CAPABILITY_DOCUMENT_MERGE, CAPABILITY_GENERATE_IMAGE, CAPABILITY_TABULAR_ANALYZE, + CAPABILITY_TABULAR_INSPECT, + CAPABILITY_TABULAR_MERGE, CAPABILITY_WORKFLOW_PROPOSE, CAPABILITY_WORKFLOW_RESULTS, CAPABILITY_WORKFLOW_RUN, @@ -76,6 +79,11 @@ ResultContractError, StepBindings, TaskResult, canonical_bytes, output_name, validate_input_bindings, ) from functions_orchestration_deliverables import DeliverableError, compile_deliverables +# The merge engine imports only the standard library; plan validation shares its option rules. +from functions_tabular_merge import ( + TABULAR_COLUMN_MAPPING_PROFILE, TabularMergeError, tabular_inspect_options_from_arguments, + tabular_merge_options_from_arguments, +) ORCHESTRATION_ELICITATION_CONTRACT_VERSION = 2 @@ -572,6 +580,85 @@ def _apply_image_input_policy(steps, existing_results): ) +def _require_merge_mapping_producers(steps): + """A merge's mapping input must come from a compose step that prepared a column mapping. + + A retained result named by alias is checked against the profile when the merge runs. + """ + by_id = {step['step_id']: step for step in steps} + for step in steps: + if step['capability_id'] != CAPABILITY_TABULAR_MERGE: + continue + for spec in step_input_specs(step): + if spec.name != 'mapping' or spec.binding.step_id is None: + continue + producer = by_id.get(spec.binding.step_id) + if producer is None: + # A missing producer is reported by the binding validation that follows. + continue + output = next( + (item for item in producer['outputs'] if item.get('name') == spec.binding.output_name), None, + ) + if ( + producer['capability_id'] != CAPABILITY_COMPOSE + or (output or {}).get('profile') != TABULAR_COLUMN_MAPPING_PROFILE + ): + raise PlanValidationError( + f'Step "{step["step_id"]}" binds a "mapping" input that is not a prepared column ' + 'mapping. Bind the output of a compose step declared as structured-v1 with profile ' + f'{TABULAR_COLUMN_MAPPING_PROFILE}.', + code='mapping_profile_required', rule='merge_mapping_profile', + ) + + +def _require_assembled_render_sources(steps, existing_results): + """A merged file renders only from a document merge's assembly, in that merge's format. + + A retained assembly named by alias is checked against its format when the file renders. + """ + # The profile and format names are plain constants; the export stack stays unloaded. + from functions_document_merge_assembly import ASSEMBLED_OUTPUT_FORMATS + from functions_generated_export_registry import ASSEMBLED_DOCUMENT_PROFILE + + by_id = {step['step_id']: step for step in steps} + for step in steps: + if step['capability_id'] != 'render_file' or step['arguments'].get('profile') != ASSEMBLED_DOCUMENT_PROFILE: + continue + sources = [spec.binding for spec in step_input_specs(step) if spec.name == 'source'] + if not sources: + # A missing source input is reported by the binding validation that follows. + continue + source = sources[0] + if source.existing_result is not None: + reference = (existing_results or {}).get(source.existing_result) + producer = getattr(getattr(reference, 'producer', None), 'capability_id', None) + if producer == CAPABILITY_DOCUMENT_MERGE and getattr(reference, 'output_name', None) == 'assembly': + continue + producer_step = None + else: + if source.step_id not in by_id: + # A missing producer is reported by the binding validation that follows. + continue + producer_step = by_id[source.step_id] + if ( + producer_step is None or producer_step['capability_id'] != CAPABILITY_DOCUMENT_MERGE + or source.output_name != 'assembly' + ): + raise PlanValidationError( + f'Step "{step["step_id"]}" renders a merged file but does not bind the "assembly" output of ' + 'a document_merge step. Bind that output, or use a profile that fits the bound source.', + code='result_kind_incompatible', rule='assembled_render_source', + ) + kind = producer_step['arguments'].get('kind') + expected = ASSEMBLED_OUTPUT_FORMATS.get(kind) + if step['arguments'].get('output_format') != expected: + raise PlanValidationError( + f'Step "{step["step_id"]}" renders a {kind} merge in another format. Use output_format ' + f'{expected} for a {kind} merge.', + code='result_kind_incompatible', rule='assembled_render_format', + ) + + def _reject_workflow_proposal_consumers(steps, final_response): """A workflow proposal is reviewed on its own card, so no step and no answer may read it.""" producers = {step['step_id'] for step in steps if step['capability_id'] == CAPABILITY_WORKFLOW_PROPOSE} @@ -873,9 +960,66 @@ def validate_dependency_plan( ) if capability_id == 'document_analyze' and arguments.get('document_ids') and 'sources' in step['inputs']: raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + if capability_id == CAPABILITY_TABULAR_MERGE: + if not arguments.get('document_ids') and 'sources' not in step['inputs']: + raise PlanValidationError( + 'Merge requires arguments.document_ids naming at least two CSV or Excel files in ' + 'merge order, or an inputs.sources source-set binding.', + code='source_binding_required', rule='merge_sources_required', + ) + if arguments.get('document_ids') and 'sources' in step['inputs']: + raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + try: + tabular_merge_options_from_arguments(arguments, mapping_bound='mapping' in step['inputs']) + except TabularMergeError as exc: + raise PlanValidationError( + f'Step "{step["step_id"]}" has merge settings that cannot be used together: ' + f'{exc.message}', + code='merge_options_invalid', rule='merge_options_invalid', + ) from exc + if capability_id == CAPABILITY_DOCUMENT_MERGE: + if not arguments.get('document_ids') and 'sources' not in step['inputs']: + raise PlanValidationError( + 'Merge documents requires arguments.document_ids naming at least two files in merge ' + 'order, or an inputs.sources source-set binding.', + code='source_binding_required', rule='merge_sources_required', + ) + if arguments.get('document_ids') and 'sources' in step['inputs']: + raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + # The document merge engine and its option rules are standard-library only. + from functions_document_merge import DocumentMergeError + from functions_document_merge_assembly import document_merge_options_from_arguments + + try: + document_merge_options_from_arguments(arguments) + except DocumentMergeError as exc: + raise PlanValidationError( + f'Step "{step["step_id"]}" has merge settings that cannot be used together: ' + f'{exc.message}', + code='merge_options_invalid', rule='merge_options_invalid', + ) from exc + if capability_id == CAPABILITY_TABULAR_INSPECT: + if not arguments.get('document_ids') and 'sources' not in step['inputs']: + raise PlanValidationError( + 'Inspect requires arguments.document_ids naming the CSV or Excel files, or an ' + 'inputs.sources source-set binding.', + code='source_binding_required', rule='inspect_sources_required', + ) + if arguments.get('document_ids') and 'sources' in step['inputs']: + raise PlanValidationError('Use either explicit document IDs or a named source-set input.') + try: + tabular_inspect_options_from_arguments(arguments) + except TabularMergeError as exc: + raise PlanValidationError( + f'Step "{step["step_id"]}" has inspection settings that cannot be used together: ' + f'{exc.message}', + code='merge_options_invalid', rule='inspect_options_invalid', + ) from exc accepted.append(step) if not any(step['enabled'] for step in accepted): raise PlanValidationError('The plan contains no enabled work.') + _require_merge_mapping_producers(accepted) + _require_assembled_render_sources(accepted, existing_results) _apply_image_input_policy(accepted, existing_results) _reject_workflow_proposal_consumers(accepted, plan.get('final_response')) _reject_workflow_run_consumers(accepted, plan.get('final_response')) @@ -1009,6 +1153,40 @@ def validate_plan_document_source_kinds(plan, document_source_kinds): if not isinstance(document_source_kinds, dict): raise PlanValidationError('Document type metadata is unavailable.', code='source_metadata_invalid') for step in plan.get('steps') or []: + if step.get('enabled', True) and step.get('capability_id') in ( + CAPABILITY_TABULAR_MERGE, CAPABILITY_TABULAR_INSPECT, + ): + if any( + document_source_kinds.get(document_id) not in (None, 'tabular') + for document_id in plan_document_ids({'steps': [step]}) + ): + action = 'merges' if step['capability_id'] == CAPABILITY_TABULAR_MERGE else 'inspects' + raise PlanValidationError( + f'Step "{step["step_id"]}" {action} only CSV or Excel files, and a selected source is ' + 'another kind of document. Use only the tabular sources there, or prepare the others ' + 'with compatible offered capabilities. Keep every selected source.', + code='source_kind_invalid', rule='tabular_source_required', + ) + continue + if step.get('enabled', True) and step.get('capability_id') == CAPABILITY_DOCUMENT_MERGE: + workbook = (step.get('arguments') or {}).get('kind') == 'workbook' + kinds = [ + document_source_kinds.get(document_id) for document_id in plan_document_ids({'steps': [step]}) + ] + if workbook and any(kind not in (None, 'tabular') for kind in kinds): + raise PlanValidationError( + f'Step "{step["step_id"]}" puts CSV or Excel files on the sheets of one workbook, and a ' + 'selected source is another kind of document. Keep every selected source.', + code='source_kind_invalid', rule='tabular_source_required', + ) + if not workbook and 'tabular' in kinds: + raise PlanValidationError( + f'Step "{step["step_id"]}" merges documents, and a selected source is a CSV or Excel ' + 'file. Use tabular_merge to append spreadsheet rows, or kind workbook to keep each ' + 'one as a sheet. Keep every selected source.', + code='source_kind_invalid', rule='document_source_required', + ) + continue if not step.get('enabled', True) or step.get('capability_id') not in ( 'document_analyze', 'document_compare', ): @@ -1531,6 +1709,60 @@ def summarize_plan(plan): 'image_generation_unavailable': 'Image generation is not available for this deployment right now.', 'image_request_invalid': 'The image model did not accept the planned image request.', 'retry_would_repeat': 'A retry would send the same request that was declined. Ask again with a different description instead.', + 'merge_schema_mismatch': ( + "The selected files don't all have the same columns, so nothing was merged. Ask to keep every " + 'column, to line up columns that have different names, or to leave out the files that ' + "don't match." + ), + 'merge_nothing_to_merge': ( + 'None of the selected files had a sheet with column headers that fit the merge, so nothing ' + 'was merged.' + ), + 'merge_columns_invalid': ( + 'A selected file has duplicate or overly long column headers, no header row, or too many ' + 'columns, so nothing was merged.' + ), + 'merge_columns_not_found': ( + "A column named for removing duplicates or sorting isn't in the merged files, so nothing " + 'was merged.' + ), + 'merge_rows_invalid': ( + 'A selected file has a row with more values than column headers, or a value longer than a ' + 'spreadsheet cell allows, so nothing was merged.' + ), + 'merge_source_unreadable': ( + "A selected file couldn't be read as a CSV or Excel file. It may be damaged or " + 'password-protected.' + ), + 'merge_sheet_not_found': "A selected workbook doesn't have the requested sheet, so nothing was merged.", + 'merge_sources_invalid': 'Merging needs at least two different CSV or Excel files (.csv, .xlsx, .xlsm or .xls).', + 'merge_options_invalid': "This merge's settings can't be used together, so nothing was merged.", + 'merge_mapping_invalid': "The column mapping prepared for this merge isn't valid, so nothing was merged.", + 'merge_limit_exceeded': ( + 'The merge is larger than chat allows. Merge fewer or smaller files, or ask for a workflow ' + 'to merge them.' + ), + 'inspect_sources_invalid': 'Only CSV and Excel files (.csv, .xlsx, .xlsm or .xls) can be inspected.', + 'inspect_limit_exceeded': ( + 'The selected files have too many sheets or columns to inspect here. Inspect fewer files at ' + 'a time.' + ), + 'document_merge_sources_invalid': ( + 'Merging needs at least two different files of the kind being merged: PDFs (.pdf), Word ' + 'documents (.docx), PowerPoint decks (.pptx), or CSV and Excel files for a workbook.' + ), + 'document_merge_source_unreadable': ( + "A selected file couldn't be read. It may be damaged, password-protected or macro-enabled, so " + 'nothing was merged.' + ), + 'document_merge_active_content': ( + 'A selected Word document has fields that start other programs (DDE), so nothing was merged.' + ), + 'document_merge_limit_exceeded': ( + 'The merged file would be larger than chat allows. Merge fewer or smaller files, or ask for a ' + 'workflow to merge them.' + ), + 'document_merge_failed': "The selected files couldn't be merged into one file.", 'step_failed': 'This operation could not complete.', 'message_not_saved': 'The explanation could not be saved. Reload this run to check its durable status.', LEGACY_PLAN_CODE: LEGACY_PLAN_MESSAGE, diff --git a/application/single_app/functions_orchestration_services.py b/application/single_app/functions_orchestration_services.py index ffd0a2382..ca92d5a66 100644 --- a/application/single_app/functions_orchestration_services.py +++ b/application/single_app/functions_orchestration_services.py @@ -1,13 +1,14 @@ # functions_orchestration_services.py """Bind initialized application resources to the retained-result runtime. -Version: 0.261.139 +Version: 0.261.245 The web and scheduler owners supply storage, current access callbacks and private artifact transport. This module never discovers configuration, credentials or Flask request state, and never initializes a client as an import side effect. Each run attempt is its own approved work, so a retry's files never reuse the -superseded attempt's output identity. +superseded attempt's output identity. A merged document's original files are read +only through the owner-supplied document reader (0.261.245). """ from copy import deepcopy @@ -28,6 +29,9 @@ InputBinding, ResultContractError, ResultRef, TaskResult, canonical_digest, ) from functions_orchestration_results import OrchestrationResultAccess, OrchestrationResults +from functions_tabular_merge import ( + TABULAR_COLUMN_MAPPING_PROFILE, TabularMergeError, tabular_column_mapping_schema, tabular_mapping_from_profile, +) from functions_workflow_result_store import _orchestration_identity @@ -35,11 +39,24 @@ def composition_profiles(): - """Use the renderer's exact prepared-content schema, not a second slide grammar.""" - return {PREPARED_SLIDE_DECK_VERSION: get_prepared_slide_deck_schema()} + """Use each owner's exact prepared-content schema, not a second grammar. + + The slide deck schema is the renderer's; the column mapping schema is the merge engine's. + """ + return { + PREPARED_SLIDE_DECK_VERSION: get_prepared_slide_deck_schema(), + TABULAR_COLUMN_MAPPING_PROFILE: tabular_column_mapping_schema(), + } def validate_composition_profile(profile, value): + if profile == TABULAR_COLUMN_MAPPING_PROFILE: + try: + tabular_mapping_from_profile(value) + except TabularMergeError as exc: + # Asking once more names the broken rule; the reply itself is never logged. + raise ResultContractError("result_schema_invalid") from exc + return True if profile != PREPARED_SLIDE_DECK_VERSION: raise ResultContractError("result_profile_unavailable") # Office libraries are execution dependencies, not registry/bootstrap dependencies. @@ -131,7 +148,7 @@ def __init__( transport, authorize_execution, max_output_bytes, external_source_catalog=None, external_source_authorizer=None, external_source_admission=None, external_source_preflight=None, native_bridge_for_step=None, - capture_external_source_configuration=None, image_asset_reader=None, + capture_external_source_configuration=None, image_asset_reader=None, document_bytes_reader=None, ): if external_source_preflight is not None and not callable(external_source_preflight): raise ResultContractError("result_external_reader_required") @@ -170,7 +187,7 @@ def __init__( self.rendering = OrchestrationRenderingService( self.outputs, self.results, transport, authorize_execution=authorize_execution, max_output_bytes=max_output_bytes, - image_asset_reader=image_asset_reader, + image_asset_reader=image_asset_reader, document_bytes_reader=document_bytes_reader, ) @property diff --git a/application/single_app/functions_orchestration_workflow_proposals.py b/application/single_app/functions_orchestration_workflow_proposals.py index 8262e451e..dd8eb4c5c 100644 --- a/application/single_app/functions_orchestration_workflow_proposals.py +++ b/application/single_app/functions_orchestration_workflow_proposals.py @@ -1,7 +1,7 @@ # functions_orchestration_workflow_proposals.py """Decisions on workflow proposals from chat orchestration: status, accept, deny and draft. -Version: 0.261.207 +Version: 0.261.242 Implemented in: 0.261.207 A plan's ``workflow_propose`` step describes a personal workflow and creates nothing @@ -158,10 +158,13 @@ _CONFLICT_DRAFT_CODES = frozenset({ 'quota_exceeded', 'workflow_conflict', 'agent_unavailable', 'reference_unknown', 'reference_unauthorized', 'file_sync_source_unavailable', 'workflow_unavailable', 'workflow_definition_conflict', 'workflow_deleted', + 'merge_unavailable', }) _INVALID_DRAFT_CODES = frozenset({ 'blueprint_invalid', 'unsupported_field', 'too_many_tasks', 'trigger_invalid', 'cadence_below_minimum', 'invalid_workflow', 'invalid_workflow_definition', 'invalid_workflow_settings', 'invalid_workflow_alerts', + 'merge_inputs_required', 'merge_trigger_required', 'merge_options_invalid', 'merge_runner_invalid', + 'merge_format_invalid', }) _INVALID_WORKFLOW_MESSAGE = 'Invalid workflow settings. Review the task, runner, trigger, and document inputs.' _SAFE_POINTER = re.compile(r'^(/[A-Za-z0-9_-]{1,64}){0,8}$') diff --git a/application/single_app/functions_orchestration_workflows.py b/application/single_app/functions_orchestration_workflows.py index 727cb49cc..952ff91e7 100644 --- a/application/single_app/functions_orchestration_workflows.py +++ b/application/single_app/functions_orchestration_workflows.py @@ -1,7 +1,7 @@ # functions_orchestration_workflows.py """Workflow proposals from chat orchestration: plan checks, the proposal step and degrading. -Version: 0.261.207 +Version: 0.261.242 Implemented in: 0.261.207 A plan proposes a personal workflow with one ``workflow_propose`` step. Its ``blueprint`` @@ -32,6 +32,7 @@ from datetime import datetime, timedelta, timezone from functions_appinsights import log_event +from functions_document_actions import MERGE_KIND_OUTPUT_FORMATS, MERGE_KIND_TABULAR from functions_mixed_source_orchestration import MixedSourceCancellationError from functions_orchestration_registry import CAPABILITY_WORKFLOW_PROPOSE from functions_orchestration_result_contracts import ( @@ -488,6 +489,7 @@ def _proposal_summary(blueprint, task_actions, planning, used, request_time_zone document_names.get(handle) or 'Document' for handle in task.get('inputs') or () if isinstance(handle, str) ], + **({'merge': _merge_summary(task['merge'])} if isinstance(task.get('merge'), dict) else {}), }) m365_sources, can_send, required = set(), False, False @@ -528,9 +530,21 @@ def _uncovered_task(blueprint, task_actions, planning): return False +def _merge_summary(merge): + """What a proposed merge task merges and creates, for the card's fixed wording.""" + kind = merge.get('kind') or MERGE_KIND_TABULAR + return { + 'kind': kind, + 'files': merge.get('files'), + 'output_format': merge.get('output_format') or MERGE_KIND_OUTPUT_FORMATS[kind][0], + } + + def _uses_default_model(blueprint): + # A merge task merges files with code, so it never needs the default model. return any( - isinstance(task, dict) and _runner_agent(task) is None for task in blueprint.get('tasks') or () + isinstance(task, dict) and _runner_agent(task) is None and not task.get('merge') + for task in blueprint.get('tasks') or () ) diff --git a/application/single_app/functions_personal_workflows.py b/application/single_app/functions_personal_workflows.py index ad1df3a3e..513d400e1 100644 --- a/application/single_app/functions_personal_workflows.py +++ b/application/single_app/functions_personal_workflows.py @@ -22,7 +22,10 @@ from functions_debug import debug_print from functions_document_actions import ( DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_CONTEXT_WORKFLOW, + MERGE_TARGET_MODE_CHANGED, + MergeActionError, build_analyze_config, get_document_action_max_documents_by_type, get_enabled_document_action_types, @@ -324,6 +327,9 @@ def _normalize_workflow_tasks( raw_document_action = default_document_action try: normalized_task['document_action'] = task_document_action_normalizer(raw_document_action) + except WorkflowPublicValidationError as exc: + # Reviewed messages stay public; the task is named by position, never by its name. + raise WorkflowPublicValidationError(f'Workflow task {index + 1}: {exc.public_message}') from exc except ValueError as exc: raise ValueError(f'Workflow task {index + 1} ({name}): {exc}') from exc @@ -355,12 +361,34 @@ def _normalize_workflow_error_handling(workflow_data, existing_workflow=None): } +def _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets): + """Merging the files a sync changed needs a File Sync trigger that passes them on.""" + if ( + isinstance(action_payload, dict) + and str(action_payload.get('type') or '').strip().lower() == DOCUMENT_ACTION_TYPE_MERGE + and str(action_payload.get('target_mode') or '').strip().lower() == MERGE_TARGET_MODE_CHANGED + and not allow_empty_file_sync_targets + ): + raise WorkflowPublicValidationError( + 'Merging the files a sync changed needs a File Sync trigger that uses changed documents.' + ) + + +def _normalize_merge_errors_for_save(normalize): + """Run a document action normalizer, showing a Merge action's reviewed reason as written.""" + try: + return normalize() + except MergeActionError as exc: + raise WorkflowPublicValidationError(str(exc)) from exc + + def _normalize_document_action_config(workflow_data, existing_workflow=None, allow_empty_file_sync_targets=False, settings=None): workflow_data = workflow_data if isinstance(workflow_data, dict) else {} existing_workflow = existing_workflow if isinstance(existing_workflow, dict) else {} settings = get_settings() if settings is None else settings action_payload = workflow_data.get('document_action') + _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets) if allow_empty_file_sync_targets and isinstance(action_payload, dict): action_type = str(action_payload.get('type') or '').strip().lower() document_ids = action_payload.get('document_ids') if isinstance(action_payload.get('document_ids'), list) else [] @@ -376,12 +404,12 @@ def _normalize_document_action_config(workflow_data, existing_workflow=None, all DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), ) normalized_action['document_ids'] = [] return normalized_action - return normalize_document_action_config( + return _normalize_merge_errors_for_save(lambda: normalize_document_action_config( action_payload=action_payload, existing_action=existing_workflow.get('document_action'), legacy_analyze=workflow_data.get('analyze') or existing_workflow.get('analyze'), @@ -389,8 +417,8 @@ def _normalize_document_action_config(workflow_data, existing_workflow=None, all DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), - ) + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), + )) def _normalize_task_document_action_config(action_payload, allow_empty_file_sync_targets=False, settings=None, @@ -398,11 +426,12 @@ def _normalize_task_document_action_config(action_payload, allow_empty_file_sync """Normalize a single workflow task's document action payload.""" source_settings = settings if isinstance(settings, dict) else get_settings() action_payload = action_payload if isinstance(action_payload, dict) else {'type': 'none'} + _require_changed_file_merge_trigger(action_payload, allow_empty_file_sync_targets) max_documents_by_type = get_document_action_max_documents_by_type( DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=source_settings, ) - allowed_action_types = get_enabled_document_action_types(settings=source_settings) + allowed_action_types = get_enabled_document_action_types(settings=source_settings, include_merge=True) if action_payload.get('target_mode') == 'current_item': if ( not allow_current_item or action_payload.get('type') != DOCUMENT_ACTION_TYPE_ANALYZE @@ -433,11 +462,11 @@ def _normalize_task_document_action_config(action_payload, allow_empty_file_sync normalized_action['document_ids'] = [] return normalized_action - return normalize_document_action_config( + return _normalize_merge_errors_for_save(lambda: normalize_document_action_config( action_payload=action_payload, max_documents_by_type=max_documents_by_type, allowed_action_types=allowed_action_types, - ) + )) def _normalize_file_sync_config(user_id, workflow_data, existing_workflow=None, sanitize_source=None): diff --git a/application/single_app/functions_structured_file_renderers.py b/application/single_app/functions_structured_file_renderers.py index 9284de908..bcc4ad9d9 100644 --- a/application/single_app/functions_structured_file_renderers.py +++ b/application/single_app/functions_structured_file_renderers.py @@ -21,7 +21,7 @@ GeneratedFileExportReadiness, GeneratedFileExportStream, ) -from functions_generated_export_registry import resolve_generated_file_export_format +from functions_generated_export_registry import EXACT_TABULAR_PROFILES, resolve_generated_file_export_format from functions_temp_files import scratch_file_dir @@ -253,6 +253,18 @@ def _validate_columns(columns, limits, checks): return tuple(columns), safe_columns +def _tabular_request_columns(source, request): + """Explicit projections name their columns; exact profiles write the retained schema in order.""" + if request.profile not in EXACT_TABULAR_PROFILES: + return request.columns + if request.columns is not None: + raise GeneratedFileExportError('invalid_options', 'Exact tabular exports write every retained column.') + columns = getattr(source, 'columns', None) + if type(columns) not in (tuple, list) or not columns: + raise GeneratedFileExportError('unsupported_source', 'Exact tabular exports need a retained column schema.') + return tuple(columns) + + def _csv_cell(value): if value is None: return '' @@ -266,7 +278,7 @@ def _csv_cell(value): def _render_csv(writer, source, request, limits, checks): - columns, safe_columns = _validate_columns(request.columns, limits, checks) + columns, safe_columns = _validate_columns(_tabular_request_columns(source, request), limits, checks) expected_keys = set(columns) csv_writer = csv.writer(writer, lineterminator='\r\n') csv_writer.writerow(safe_columns) diff --git a/application/single_app/functions_tabular_merge.py b/application/single_app/functions_tabular_merge.py new file mode 100644 index 000000000..b652104ba --- /dev/null +++ b/application/single_app/functions_tabular_merge.py @@ -0,0 +1,2297 @@ +# functions_tabular_merge.py +"""Deterministic merging and inspection of CSV and Excel files. + +Version: 0.261.248 +Implemented in: 0.261.239 +Reconciliation policies, sheet modes, duplicate removal, sorting and inspection added in: 0.261.240 +Single-file merges for workflow files found at run time (min_sources=1) added in: 0.261.241 +Workbooks with unsafe or unreadable XML refused in: 0.261.245 +Spooled rows written to the container's scratch directory in: 0.261.247 +CodeQL cleanup for intentionally ignored reader cleanup failures added in: 0.261.248 + +The engine is pure: it receives already-authorized byte loaders, never resolves +documents, settings, storage, routes, or models, and performs no model work. +Values are kept as text so leading zeros, codes, and identifiers survive exactly. +Rows are spooled to a bounded temporary file during the single parse, so callers +learn the exact row count before they persist or render the merged result. + +Schema policies decide which columns the merged table has: + +- ``by_name``: every file has the same columns, in any order (the first file's spelling is kept). +- ``exact_order``: every file has the same columns in the same order. +- ``union``: every column from every file; a file without a column leaves it blank (null). +- ``mapped``: exactly the requested ``columns``; other columns are left out and reported. + +``column_aliases`` rename alternative headers to one target name before matching, under +every policy. Files whose columns do not fit fail the merge, or are left out and reported +when ``on_incompatible`` is ``exclude``. +""" + +import bisect +import codecs +import csv +from array import array +from dataclasses import dataclass, field +from datetime import date, datetime, time, timedelta, timezone +import hashlib +import io +import json +import math +import os +import re +import tempfile +from typing import Callable, List, Mapping, Optional, Sequence, Tuple +import unicodedata +import zipfile + +from functions_ooxml_package_guard import UnreadablePackageError, first_unsafe_xml_part +from functions_temp_files import scratch_file_dir + +TABULAR_MERGE_REPORT_VERSION = "tabular-merge-report-v1" +TABULAR_INSPECTION_VERSION = "tabular-inspection-v1" + +SCHEMA_POLICY_BY_NAME = "by_name" +SCHEMA_POLICY_EXACT_ORDER = "exact_order" +SCHEMA_POLICY_UNION = "union" +SCHEMA_POLICY_MAPPED = "mapped" +TABULAR_MERGE_SCHEMA_POLICIES = ( + SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER, SCHEMA_POLICY_UNION, SCHEMA_POLICY_MAPPED, +) + +SHEETS_FIRST = "first" +SHEETS_ALL = "all" +TABULAR_MERGE_SHEET_MODES = (SHEETS_FIRST, SHEETS_ALL) + +ON_INCOMPATIBLE_FAIL = "fail" +ON_INCOMPATIBLE_EXCLUDE = "exclude" +TABULAR_MERGE_INCOMPATIBLE_MODES = (ON_INCOMPATIBLE_FAIL, ON_INCOMPATIBLE_EXCLUDE) + +DEDUPE_NONE = "none" +DEDUPE_EXACT_ROWS = "exact_rows" +DEDUPE_KEY_COLUMNS = "key_columns" +TABULAR_MERGE_DEDUPE_MODES = (DEDUPE_NONE, DEDUPE_EXACT_ROWS, DEDUPE_KEY_COLUMNS) +DEDUPE_KEEP_FIRST = "first" +DEDUPE_KEEP_LAST = "last" +TABULAR_MERGE_DEDUPE_KEEP = (DEDUPE_KEEP_FIRST, DEDUPE_KEEP_LAST) + +SORT_TYPE_TEXT = "text" +SORT_TYPE_NUMBER = "number" +SORT_TYPE_DATE = "date" +TABULAR_MERGE_SORT_TYPES = (SORT_TYPE_TEXT, SORT_TYPE_NUMBER, SORT_TYPE_DATE) + +TABULAR_MERGE_FORMAT_CSV = "csv" +TABULAR_MERGE_FORMAT_XLSX = "xlsx" +TABULAR_MERGE_FORMAT_XLS = "xls" +TABULAR_MERGE_EXTENSION_FORMATS = { + ".csv": TABULAR_MERGE_FORMAT_CSV, + ".xlsx": TABULAR_MERGE_FORMAT_XLSX, + ".xlsm": TABULAR_MERGE_FORMAT_XLSX, + ".xls": TABULAR_MERGE_FORMAT_XLS, +} +TABULAR_MERGE_SUPPORTED_EXTENSIONS = tuple(TABULAR_MERGE_EXTENSION_FORMATS) + +DEFAULT_SOURCE_COLUMN_NAME = "Source File" +DEFAULT_SHEET_COLUMN_NAME = "Source Sheet" +MAX_SOURCE_COLUMN_NAME_CHARS = 128 +MAX_SHEET_NAME_CHARS = 31 +MAX_COLUMN_NAME_BYTES = 256 +MAX_HEADER_ROW = 1000 +MAX_MAPPED_COLUMNS = 255 +MAX_ALIAS_TARGETS = 256 +MAX_ALIASES_PER_TARGET = 32 +MAX_DEDUPE_COLUMNS = 16 +MAX_SORT_COLUMNS = 3 +DEFAULT_INSPECT_SAMPLE_ROWS = 3 +MAX_INSPECT_SAMPLE_ROWS = 10 + +SOURCE_STATUS_MERGED = "merged" +SOURCE_STATUS_COMPATIBLE = "compatible" +SOURCE_STATUS_SCHEMA_MISMATCH = "schema_mismatch" +SOURCE_STATUS_EXCLUDED = "excluded" +SOURCE_STATUS_SKIPPED = "skipped" +SOURCE_STATUS_NOT_PROCESSED = "not_processed" + +_CSV_DELIMITERS = (",", ";", "\t", "|") +_CSV_SNIFF_BYTES = 64 * 1024 +_SPOOL_MEMORY_BYTES = 8 * 1024 * 1024 +_CANCEL_CHECK_INTERVAL = 1000 +_ZIP_MAGIC = b"PK\x03\x04" +_OLE_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +_MAX_EXACT_INTEGER_FLOAT = float(2 ** 53) +_INSPECT_CELL_CHARS = 80 +_INSPECT_MAX_BYTES = 96 * 1024 +_NUMBER_TEXT = re.compile( + r"[+-]?(?:(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?|\.\d+)(?:[eE][+-]?\d+)?" +) +_CURRENCY_PREFIXES = "$\u20ac\u00a3\u00a5" +_NOT_WORD = re.compile(r"[\W_]+") + +_EXCEL_LIMITATIONS = ( + "Excel formulas are merged as their last calculated values; a workbook saved without " + "calculated values merges those cells as blanks.", + "Excel numbers, booleans and dates are written as plain text, with dates in ISO 8601 form.", +) +_EXCLUDED_LIMITATION = ( + "{count} of {total} file(s) or sheet(s) were left out because their columns don't fit the " + "merge; the merge report lists each one and why." +) +_BLANK_COLUMNS_LIMITATION = ( + "Columns that some files don't have are left blank for those files' rows; the merge report " + "lists the missing columns." +) +_IGNORED_COLUMNS_LIMITATION = ( + "Columns that aren't in the requested column list were left out; the merge report lists them." +) + + +class TabularMergeError(ValueError): + """A safe, user-presentable merge failure with a stable code.""" + + def __init__(self, code, message, *, report=None, details=None): + super().__init__(message) + self.code = code + self.message = message + self.report = report + self.details = dict(details or {}) + + +class TabularMergeCancelled(TabularMergeError): + """The caller asked the merge to stop.""" + + def __init__(self): + super().__init__("cancelled", "The merge was cancelled.") + + +@dataclass(frozen=True) +class TabularMergeLimits: + """Bounds for one merge; every value must be a positive integer.""" + + max_sources: int = 10 + max_total_rows: int = 250_000 + max_source_bytes: int = 100 * 1024 * 1024 + max_columns: int = 256 + max_header_bytes: int = 256 + max_cell_chars: int = 32_767 + max_workbook_uncompressed_bytes: int = 512 * 1024 * 1024 + max_workbook_entries: int = 10_000 + max_sort_rows: int = 250_000 + max_sheets_per_source: int = 100 + + def __post_init__(self): + for name in ( + "max_sources", "max_total_rows", "max_source_bytes", "max_columns", + "max_header_bytes", "max_cell_chars", "max_workbook_uncompressed_bytes", + "max_workbook_entries", "max_sort_rows", "max_sheets_per_source", + ): + value = getattr(self, name) + if type(value) is not int or value < 1: + raise ValueError(f"{name} must be a positive integer.") + if self.max_sources < 2: + raise ValueError("max_sources must allow at least two sources.") + + +@dataclass(frozen=True) +class TabularMergeSort: + """One sort key: a merged column, its direction and how its values compare.""" + + column: str + descending: bool = False + value_type: str = SORT_TYPE_TEXT + + def __post_init__(self): + name = _display_header(self.column) if isinstance(self.column, str) else "" + if not name or len(name.encode("utf-8")) > MAX_COLUMN_NAME_BYTES: + raise TabularMergeError("invalid_options", "Each sort key needs a column name.") + object.__setattr__(self, "column", name) + if type(self.descending) is not bool: + raise TabularMergeError("invalid_options", "A sort direction must be true or false.") + if self.value_type not in TABULAR_MERGE_SORT_TYPES: + raise TabularMergeError( + "invalid_options", f"A sort type must be one of: {', '.join(TABULAR_MERGE_SORT_TYPES)}.", + ) + + def to_dict(self): + return {"column": self.column, "descending": self.descending, "value_type": self.value_type} + + +@dataclass(frozen=True) +class TabularMergeOptions: + """How sources are matched, combined, de-duplicated, ordered and attributed.""" + + schema_policy: str = SCHEMA_POLICY_BY_NAME + sheet: Optional[str] = None + include_source_column: bool = True + source_column_name: str = DEFAULT_SOURCE_COLUMN_NAME + sheets: str = SHEETS_FIRST + header_row: Optional[int] = None + columns: Tuple[str, ...] = () + column_aliases: Tuple[Tuple[str, Tuple[str, ...]], ...] = () + on_incompatible: str = ON_INCOMPATIBLE_FAIL + dedupe: str = DEDUPE_NONE + dedupe_columns: Tuple[str, ...] = () + dedupe_keep: str = DEDUPE_KEEP_FIRST + sort_by: Tuple[TabularMergeSort, ...] = () + + def __post_init__(self): + if self.schema_policy not in TABULAR_MERGE_SCHEMA_POLICIES: + raise TabularMergeError( + "invalid_options", + f"Schema policy must be one of: {', '.join(TABULAR_MERGE_SCHEMA_POLICIES)}.", + ) + _validate_table_selection(self.sheet, self.sheets, self.header_row) + if type(self.include_source_column) is not bool: + raise TabularMergeError("invalid_options", "include_source_column must be true or false.") + name = _display_header(self.source_column_name) if isinstance(self.source_column_name, str) else "" + if not name or len(name) > MAX_SOURCE_COLUMN_NAME_CHARS or len(name.encode("utf-8")) > 240: + raise TabularMergeError( + "invalid_options", + f"The source column name must be 1 to {MAX_SOURCE_COLUMN_NAME_CHARS} characters.", + ) + + columns = _column_names(self.columns, "The requested columns", maximum=MAX_MAPPED_COLUMNS) + object.__setattr__(self, "columns", columns) + if self.schema_policy == SCHEMA_POLICY_MAPPED and not columns: + raise TabularMergeError("invalid_options", "The mapped policy needs the list of columns to keep.") + if self.schema_policy != SCHEMA_POLICY_MAPPED and columns: + raise TabularMergeError("invalid_options", "A list of columns is used only by the mapped policy.") + + aliases = _alias_pairs(self.column_aliases) + object.__setattr__(self, "column_aliases", aliases) + if self.schema_policy == SCHEMA_POLICY_MAPPED: + targets = {normalize_header_name(column) for column in columns} + if any(normalize_header_name(target) not in targets for target, _ in aliases): + raise TabularMergeError( + "invalid_options", "Every alias must rename a header to one of the requested columns.", + ) + + if self.on_incompatible not in TABULAR_MERGE_INCOMPATIBLE_MODES: + raise TabularMergeError( + "invalid_options", + f"on_incompatible must be one of: {', '.join(TABULAR_MERGE_INCOMPATIBLE_MODES)}.", + ) + if self.dedupe not in TABULAR_MERGE_DEDUPE_MODES: + raise TabularMergeError( + "invalid_options", f"dedupe must be one of: {', '.join(TABULAR_MERGE_DEDUPE_MODES)}.", + ) + if self.dedupe_keep not in TABULAR_MERGE_DEDUPE_KEEP: + raise TabularMergeError( + "invalid_options", f"dedupe_keep must be one of: {', '.join(TABULAR_MERGE_DEDUPE_KEEP)}.", + ) + dedupe_columns = _column_names( + self.dedupe_columns, "The duplicate key columns", maximum=MAX_DEDUPE_COLUMNS, + ) + object.__setattr__(self, "dedupe_columns", dedupe_columns) + if self.dedupe == DEDUPE_KEY_COLUMNS and not dedupe_columns: + raise TabularMergeError("invalid_options", "Removing duplicates by key needs the key columns.") + if self.dedupe != DEDUPE_KEY_COLUMNS and dedupe_columns: + raise TabularMergeError("invalid_options", "Key columns are used only when removing duplicates by key.") + + sort_by = tuple( + item if isinstance(item, TabularMergeSort) else _sort_from_mapping(item) + for item in (self.sort_by or ()) + ) + if len(sort_by) > MAX_SORT_COLUMNS: + raise TabularMergeError("invalid_options", f"Sort by at most {MAX_SORT_COLUMNS} columns.") + if len({normalize_header_name(item.column) for item in sort_by}) != len(sort_by): + raise TabularMergeError("invalid_options", "Each column can be sorted on only once.") + object.__setattr__(self, "sort_by", sort_by) + + def policy_report(self): + """The options as plain data for the merge report.""" + return { + "schema_policy": self.schema_policy, + "sheet": self.sheet, + "sheets": self.sheets, + "header_row": self.header_row, + "include_source_column": self.include_source_column, + "source_column_name": _display_header(self.source_column_name) if self.include_source_column else None, + "columns": list(self.columns), + "column_aliases": {target: list(aliases) for target, aliases in self.column_aliases}, + "on_incompatible": self.on_incompatible, + "dedupe": self.dedupe, + "dedupe_columns": list(self.dedupe_columns), + "dedupe_keep": self.dedupe_keep if self.dedupe != DEDUPE_NONE else None, + "sort_by": [item.to_dict() for item in self.sort_by], + } + + +@dataclass(frozen=True) +class TabularInspectOptions: + """Which sheets and header row to inspect, and how many sample rows to keep.""" + + sheet: Optional[str] = None + sheets: str = SHEETS_FIRST + header_row: Optional[int] = None + sample_rows: int = DEFAULT_INSPECT_SAMPLE_ROWS + + def __post_init__(self): + _validate_table_selection(self.sheet, self.sheets, self.header_row) + if type(self.sample_rows) is not int or not 0 <= self.sample_rows <= MAX_INSPECT_SAMPLE_ROWS: + raise TabularMergeError( + "invalid_options", f"Sample rows must be a whole number from 0 to {MAX_INSPECT_SAMPLE_ROWS}.", + ) + + +@dataclass(frozen=True) +class TabularMergeSource: + """One authorized source. ``load_bytes`` returns the original file bytes.""" + + source_id: str + file_name: str + load_bytes: Callable[[], bytes] = field(repr=False) + extension: str = "" + + def resolved_extension(self): + extension = (self.extension or os.path.splitext(self.file_name or "")[1]).strip().lower() + if extension and not extension.startswith("."): + extension = f".{extension}" + return extension + + +def _validate_table_selection(sheet, sheets, header_row): + if sheet is not None: + if not isinstance(sheet, str) or not sheet.strip(): + raise TabularMergeError("invalid_options", "A sheet name must be nonempty text.") + if len(sheet) > MAX_SHEET_NAME_CHARS: + raise TabularMergeError( + "invalid_options", f"A sheet name has at most {MAX_SHEET_NAME_CHARS} characters.", + ) + if sheets not in TABULAR_MERGE_SHEET_MODES: + raise TabularMergeError( + "invalid_options", f"sheets must be one of: {', '.join(TABULAR_MERGE_SHEET_MODES)}.", + ) + if sheet is not None and sheets == SHEETS_ALL: + raise TabularMergeError("invalid_options", "Name one sheet, or read all sheets, but not both.") + if header_row is not None and (type(header_row) is not int or not 1 <= header_row <= MAX_HEADER_ROW): + raise TabularMergeError( + "invalid_options", f"The header row must be a whole number from 1 to {MAX_HEADER_ROW}.", + ) + + +def _column_name(value, label): + name = _display_header(value) if isinstance(value, str) else "" + if not name or len(name.encode("utf-8")) > MAX_COLUMN_NAME_BYTES: + raise TabularMergeError( + "invalid_options", f"{label} must be nonempty names of at most {MAX_COLUMN_NAME_BYTES} bytes.", + ) + return name + + +def _column_names(values, label, *, maximum): + if values is None: + return () + if isinstance(values, (str, bytes)) or not isinstance(values, (list, tuple)): + raise TabularMergeError("invalid_options", f"{label} must be a list of column names.") + names = tuple(_column_name(value, label) for value in values) + if len(names) > maximum: + raise TabularMergeError("invalid_options", f"{label} can name at most {maximum} columns.") + if len({name.casefold() for name in names}) != len(names): + raise TabularMergeError("invalid_options", f"{label} name a column more than once.") + return names + + +def _alias_pairs(value): + """Validated ``((target, (alias, ...)), ...)``; identity aliases are dropped.""" + if not value: + return () + if isinstance(value, Mapping): + items = list(value.items()) + elif isinstance(value, (list, tuple)): + items = list(value) + else: + raise TabularMergeError("invalid_options", "Column aliases must map each column to its other names.") + if len(items) > MAX_ALIAS_TARGETS: + raise TabularMergeError("invalid_options", f"Column aliases can rename at most {MAX_ALIAS_TARGETS} columns.") + pairs = [] + targets = {} + for item in items: + if not isinstance(item, (list, tuple)) or len(item) != 2: + raise TabularMergeError("invalid_options", "Column aliases must map each column to its other names.") + target, aliases = item + target = _column_name(target, "Alias targets") + if target.casefold() in targets: + raise TabularMergeError("invalid_options", "Column aliases name a target column more than once.") + if isinstance(aliases, (str, bytes)) or not isinstance(aliases, (list, tuple)) or not aliases: + raise TabularMergeError("invalid_options", "Each aliased column needs a list of its other names.") + if len(aliases) > MAX_ALIASES_PER_TARGET: + raise TabularMergeError( + "invalid_options", f"A column can have at most {MAX_ALIASES_PER_TARGET} other names.", + ) + names = [] + for alias in aliases: + name = _column_name(alias, "Column aliases") + if name.casefold() != target.casefold() and name.casefold() not in {item.casefold() for item in names}: + names.append(name) + targets[target.casefold()] = names + pairs.append((target, tuple(names))) + seen = {} + for target, names in pairs: + for name in names: + key = name.casefold() + if key in targets: + raise TabularMergeError( + "invalid_options", f'"{name}" is both a column name and another column\'s alias.', + ) + if key in seen and seen[key] != target.casefold(): + raise TabularMergeError("invalid_options", f'"{name}" is an alias of more than one column.') + seen[key] = target.casefold() + return tuple((target, names) for target, names in pairs if names) + + +def _sort_from_mapping(value): + if not isinstance(value, Mapping): + raise TabularMergeError("invalid_options", "Each sort key must name a column.") + unknown = set(value) - {"column", "descending", "value_type"} + if unknown: + raise TabularMergeError("invalid_options", "A sort key has an unsupported setting.") + return TabularMergeSort( + column=value.get("column"), + descending=value.get("descending", False), + value_type=value.get("value_type", SORT_TYPE_TEXT), + ) + + +def tabular_merge_format_for_file_name(file_name): + """The merge reader format for a file name, or None when unsupported.""" + extension = os.path.splitext(str(file_name or ""))[1].strip().lower() + return TABULAR_MERGE_EXTENSION_FORMATS.get(extension) + + +def tabular_merge_options_from_arguments(arguments, *, mapping=None, mapping_bound=False): + """Engine options from plan arguments; a bound column mapping makes the policy ``mapped``. + + At planning time the mapping is not prepared yet, so ``mapping_bound`` checks that the + other arguments can be combined with one. Invalid combinations raise ``invalid_options``. + """ + arguments = arguments if isinstance(arguments, Mapping) else {} + policy = arguments.get("schema_policy") or SCHEMA_POLICY_BY_NAME + columns = arguments.get("columns") or () + aliases = arguments.get("column_aliases") or () + if mapping is not None or mapping_bound: + if columns or aliases: + raise TabularMergeError( + "invalid_options", "Use either a bound column mapping or explicit columns and aliases, not both.", + ) + if policy not in (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_MAPPED): + raise TabularMergeError("invalid_options", "A bound column mapping uses the mapped schema policy.") + policy = SCHEMA_POLICY_MAPPED + if mapping is None: + # The prepared mapping supplies the real columns when the merge runs. + columns = ("Mapped column",) + else: + columns = mapping["columns"] + aliases = mapping["column_aliases"] + return TabularMergeOptions( + schema_policy=policy, + sheet=arguments.get("sheet") or None, + include_source_column=arguments.get("include_source_column", True) is not False, + source_column_name=arguments.get("source_column_name") or DEFAULT_SOURCE_COLUMN_NAME, + sheets=arguments.get("sheets") or SHEETS_FIRST, + header_row=arguments.get("header_row"), + columns=columns, + column_aliases=aliases, + on_incompatible=arguments.get("on_incompatible") or ON_INCOMPATIBLE_FAIL, + dedupe=arguments.get("dedupe") or DEDUPE_NONE, + dedupe_columns=arguments.get("dedupe_columns") or (), + dedupe_keep=arguments.get("dedupe_keep") or DEDUPE_KEEP_FIRST, + sort_by=arguments.get("sort_by") or (), + ) + + +def tabular_inspect_options_from_arguments(arguments): + """Inspection options from plan arguments.""" + arguments = arguments if isinstance(arguments, Mapping) else {} + sample_rows = arguments.get("sample_rows") + return TabularInspectOptions( + sheet=arguments.get("sheet") or None, + sheets=arguments.get("sheets") or SHEETS_FIRST, + header_row=arguments.get("header_row"), + sample_rows=DEFAULT_INSPECT_SAMPLE_ROWS if sample_rows is None else sample_rows, + ) + + +# --------------------------------------------------------------------------- +# Prepared column mappings +# --------------------------------------------------------------------------- + +TABULAR_COLUMN_MAPPING_PROFILE = "tabular_column_mapping_v1" +MAPPING_CONFIDENCE_LEVELS = ("high", "medium", "low") +MAX_MAPPING_ENTRIES = 1024 +MAX_MAPPING_NOTES_CHARS = 2000 + + +def tabular_column_mapping_schema(): + """The prepared-content profile a compose step fills to line up differently named columns.""" + name = {"type": "string", "minLength": 1, "maxLength": MAX_COLUMN_NAME_BYTES} + return { + "type": "object", + "description": ( + "How to line up the columns of several CSV or Excel files before merging them. columns " + "is the merged table's columns in order; use the clearest existing header for each. " + "mappings lists every header whose name differs from the column it fills, and every " + "header to leave out (target_column null). A header that already matches a column, " + "ignoring case and spacing, needs no entry. Map a header only when it holds the same " + "kind of value; when unsure, keep it as its own column or mark confidence low." + ), + "properties": { + "columns": { + "type": "array", "items": dict(name), "minItems": 1, "maxItems": MAX_MAPPED_COLUMNS, + "uniqueItems": True, + }, + "mappings": { + "type": "array", + "maxItems": MAX_MAPPING_ENTRIES, + "items": { + "type": "object", + "properties": { + "source_column": dict(name, description="A header exactly as it appears in a file."), + "target_column": { + "type": ["string", "null"], "maxLength": MAX_COLUMN_NAME_BYTES, + "description": "The column in columns this header fills, or null to leave it out.", + }, + "confidence": {"type": "string", "enum": list(MAPPING_CONFIDENCE_LEVELS)}, + }, + "required": ["source_column", "target_column", "confidence"], + "additionalProperties": False, + }, + }, + "notes": { + "type": "string", "maxLength": MAX_MAPPING_NOTES_CHARS, + "description": "A short explanation of the mapping for the user.", + }, + }, + "required": ["columns", "mappings"], + "additionalProperties": False, + } + + +def tabular_mapping_from_profile(value): + """Validate a prepared ``tabular_column_mapping_v1`` value; return merge columns and aliases. + + Raises ``invalid_mapping`` when the value breaks the profile: unknown fields, repeated + columns or headers, a target that isn't a listed column, or a header that is itself a + different listed column. + """ + def invalid(message): + return TabularMergeError("invalid_mapping", message) + + if not isinstance(value, Mapping) or set(value) - {"columns", "mappings", "notes"}: + raise invalid("A column mapping has only columns, mappings and notes.") + raw_columns = value.get("columns") + raw_mappings = value.get("mappings") + notes = value.get("notes", "") + if not isinstance(raw_columns, list) or not 1 <= len(raw_columns) <= MAX_MAPPED_COLUMNS: + raise invalid(f"A column mapping lists 1 to {MAX_MAPPED_COLUMNS} columns.") + if not isinstance(raw_mappings, list) or len(raw_mappings) > MAX_MAPPING_ENTRIES: + raise invalid(f"A column mapping has at most {MAX_MAPPING_ENTRIES} mappings.") + if not isinstance(notes, str) or len(notes) > MAX_MAPPING_NOTES_CHARS: + raise invalid(f"Mapping notes have at most {MAX_MAPPING_NOTES_CHARS} characters.") + try: + columns = _column_names(raw_columns, "The mapped columns", maximum=MAX_MAPPED_COLUMNS) + except TabularMergeError as exc: + raise invalid(exc.message) from exc + targets = {column.casefold(): column for column in columns} + aliases = {} + ignored = [] + low_confidence = [] + sources = set() + for item in raw_mappings: + if not isinstance(item, Mapping) or set(item) != {"source_column", "target_column", "confidence"}: + raise invalid("Each mapping has exactly source_column, target_column and confidence.") + if item["confidence"] not in MAPPING_CONFIDENCE_LEVELS: + raise invalid(f"Mapping confidence is one of: {', '.join(MAPPING_CONFIDENCE_LEVELS)}.") + try: + source = _column_name(item["source_column"], "Mapped headers") + except TabularMergeError as exc: + raise invalid(exc.message) from exc + key = source.casefold() + if key in sources: + raise invalid(f'"{source}" is mapped more than once.') + sources.add(key) + target = item["target_column"] + if target is None: + if key in targets: + raise invalid(f'"{source}" is a listed column, so it can\'t be left out.') + ignored.append(source) + continue + target_key = _display_header(target).casefold() if isinstance(target, str) else "" + if target_key not in targets: + raise invalid(f'"{source}" is mapped to a column that isn\'t listed.') + if key == target_key: + continue + if key in targets: + raise invalid(f'"{source}" is a listed column, so it can\'t also fill "{targets[target_key]}".') + aliases.setdefault(targets[target_key], []).append(source) + if item["confidence"] == "low": + low_confidence.append({"source_column": source, "target_column": targets[target_key]}) + for target, names in aliases.items(): + if len(names) > MAX_ALIASES_PER_TARGET: + raise invalid(f'At most {MAX_ALIASES_PER_TARGET} headers can fill "{target}".') + return { + "columns": columns, + "column_aliases": aliases, + "ignored": ignored, + "low_confidence": low_confidence, + "notes": notes.strip(), + } + + +def iter_csv_rows(content, file_name="A selected file"): + """Decode CSV bytes; return ``(encoding, delimiter, rows)`` where rows yields lists of text.""" + source = TabularMergeSource(file_name, file_name, lambda: content) + text, encoding = _decode_csv(content, source) + delimiter = _detect_delimiter(text) + reader = csv.reader( + io.StringIO(text, newline=""), delimiter=delimiter, quotechar='"', doublequote=True, strict=False, + ) + + def rows(): + try: + for values in reader: + yield values + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} couldn't be read as CSV near line {reader.line_num}.", + ) from exc + + return encoding, delimiter, rows() + + +def normalize_header_name(value): + """The comparison key for a header: NFC, collapsed whitespace, casefolded.""" + return _display_header(value).casefold() + + +def squash_header_name(value): + """A looser key that also ignores spaces, punctuation and underscores, for near matches.""" + return _NOT_WORD.sub("", normalize_header_name(value)) + + +def _display_header(value): + text = unicodedata.normalize("NFC", str(value if value is not None else "")) + return " ".join(text.split()) + + +# --------------------------------------------------------------------------- +# Spooled rows and the merge result +# --------------------------------------------------------------------------- + + +class _RowSpool: + """Validated rows as compact JSON lines; small merges stay in memory.""" + + def __init__(self): + # The container's dedicated scratch directory, or the platform default; never the + # working directory, which the container's non-root user can't write. + self._file = tempfile.SpooledTemporaryFile(max_size=_SPOOL_MEMORY_BYTES, mode="w+b", dir=scratch_file_dir()) + self._offsets = array("q") + self._end = 0 + self.count = 0 + self.closed = False + + def append(self, values): + data = json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + b"\n" + self._file.seek(self._end) + self._file.write(data) + self._offsets.append(self._end) + self._end += len(data) + self.count += 1 + + def _read(self, index): + self._file.seek(self._offsets[index]) + line = self._file.readline() + if not line: + raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") + return json.loads(line) + + def iter_indexed(self, order=None): + """``(index, values)`` in spool order, or in ``order`` when given.""" + if self.closed: + raise TabularMergeError("result_closed", "The merged rows are no longer available.") + self._file.flush() + indices = range(self.count) if order is None else order + for index in indices: + if self.closed: + raise TabularMergeError("result_closed", "The merged rows are no longer available.") + yield index, self._read(index) + + def close(self): + if not self.closed: + self.closed = True + self._file.close() + + +class TabularMergeResult: + """The merged table: ordered columns, an exact row count, rows and a report. + + Rows can be iterated more than once until ``close()``; each pass verifies the + count. Use it as a context manager so the spooled rows are always released. + """ + + def __init__(self, columns, report, spool, *, nullable=(), row_count=None, order=None, keep=None): + self._columns = tuple(columns) + self._nullable = frozenset(nullable) + self._report = report + self._spool = spool + self._row_count = spool.count if row_count is None else row_count + self._order = order + self._keep = keep + + @property + def columns(self): + return self._columns + + @property + def nullable_columns(self): + """Columns that hold null for rows from files that don't have them.""" + return self._nullable + + @property + def row_count(self): + return self._row_count + + @property + def report(self): + return json.loads(json.dumps(self._report)) + + def iter_rows(self): + width = len(self._columns) + yielded = 0 + for index, values in self._spool.iter_indexed(self._order): + if len(values) < width: + values.extend([None] * (width - len(values))) + if self._keep is not None and not self._keep(index, values): + continue + yielded += 1 + yield values + if yielded != self._row_count: + raise TabularMergeError("count_mismatch", "The merged rows could not be read back completely.") + + def iter_records(self): + columns = self._columns + for values in self.iter_rows(): + yield dict(zip(columns, values)) + + def close(self): + self._spool.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + self.close() + return False + + +# --------------------------------------------------------------------------- +# Merging +# --------------------------------------------------------------------------- + + +def merge_tabular_sources( + sources: Sequence[TabularMergeSource], + *, + options: Optional[TabularMergeOptions] = None, + limits: Optional[TabularMergeLimits] = None, + cancel_requested: Optional[Callable[[], bool]] = None, + on_progress: Optional[Callable[[dict], None]] = None, + min_sources: int = 2, +) -> TabularMergeResult: + """Append the rows of the sources, in the given order, into one table. + + The first file (or sheet) defines the column order for ``by_name`` and ``exact_order``; + ``union`` adds each new column the first time a file has it; ``mapped`` uses exactly + the requested columns. Under ``on_incompatible="fail"`` a mismatch reads every remaining + header so the report lists all mismatches, then nothing is merged. ``min_sources`` is 1 + only for a scheduled merge of whatever files arrived, which may be a single file. + """ + if min_sources not in (1, 2): + raise ValueError("min_sources must be 1 or 2.") + options = options or TabularMergeOptions() + limits = limits or TabularMergeLimits() + sources = list(sources or ()) + _validate_sources(sources, limits, minimum=min_sources) + merger = _Merger(options, limits, len(sources), cancel_requested) + try: + for index, source in enumerate(sources): + _check_cancel(cancel_requested) + rows = merger.add_source(source) + if on_progress is not None: + on_progress({ + "index": index + 1, "total": len(sources), "file_name": source.file_name, "rows": rows, + }) + return merger.finish() + except BaseException: + merger.discard() + raise + + +class _Merger: + """The single-pass state of one merge.""" + + def __init__(self, options, limits, source_count, cancel_requested): + self.options = options + self.limits = limits + self.cancel_requested = cancel_requested + self.report = _new_report(options, source_count) + self.alias_index = _alias_index(options) + self.provenance = None + self.data_display = [] + self.data_keys = [] + self.data_index = {} + self.seen_counts = [] + self.reference = None + self.mismatch = False + self.merged_tables = 0 + self.excluded_tables = 0 + self.rows_read = 0 + self.blank_rows = 0 + self.duplicates_removed = 0 + self.sheet_null = False + self.dedupe_keys = [normalize_header_name(name) for name in options.dedupe_columns] + self.sort_keys = [normalize_header_name(item.column) for item in options.sort_by] + self.seen_digests = set() + self.last_digest = {} + self.row_starts = [] + self.row_entries = [] + self.sort_values = [] + if options.schema_policy == SCHEMA_POLICY_MAPPED: + for name in options.columns: + self._add_data_column(name, normalize_header_name(name)) + self._init_provenance(set(self.data_keys)) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + # Created last, so a refused configuration leaves nothing to release. + self.spool = _RowSpool() + + def discard(self): + self.spool.close() + + def add_source(self, source): + content = _load_source_bytes(source, self.limits) + reader = _open_reader(source, content, self.limits) + rows = 0 + try: + try: + plan = reader.plan(self.options, self.limits) + except TabularMergeError as exc: + if exc.code != "sheet_not_found" or self.options.on_incompatible != ON_INCOMPATIBLE_EXCLUDE: + raise + entry = _new_source_entry(source, reader, self.options.sheet) + self.report["sources"].append(entry) + self._incompatible(entry, "sheet_not_found") + return 0 + for sheet, skip_reason in plan: + _check_cancel(self.cancel_requested) + entry = _new_source_entry(source, reader, sheet) + self.report["sources"].append(entry) + if skip_reason is not None: + entry["status"] = SOURCE_STATUS_SKIPPED + entry["reason"] = skip_reason + continue + try: + table = reader.open_table(sheet, self.options.header_row) + except TabularMergeError as exc: + if exc.code != "empty_source" or isinstance(exc, TabularMergeCancelled): + raise + if self.options.sheets == SHEETS_ALL: + entry["status"] = SOURCE_STATUS_SKIPPED + entry["reason"] = "no_header_row" + continue + if self.options.on_incompatible != ON_INCOMPATIBLE_EXCLUDE: + raise + self._incompatible(entry, "no_header_row") + continue + try: + entry["header_row"] = table.header_row + self._add_table(source, table, entry) + finally: + table.close() + rows += entry["rows"] + finally: + reader.close() + return rows + + # -- columns ----------------------------------------------------------- + + def _add_data_column(self, display, key): + self.data_index[key] = len(self.data_keys) + self.data_keys.append(key) + self.data_display.append(display) + self.seen_counts.append(0) + + def _init_provenance(self, taken_keys): + names = [] + if self.options.include_source_column: + taken = set(taken_keys) + names.append(self._claim_provenance_name( + _display_header(self.options.source_column_name), taken, "source_column_renamed", + "file name column", + )) + if self.options.sheets == SHEETS_ALL: + taken.add(names[0].casefold()) + names.append(self._claim_provenance_name( + DEFAULT_SHEET_COLUMN_NAME, taken, "sheet_column_renamed", "sheet name column", + )) + self.provenance = names + self.report["source_column"] = names[0] if names else None + self.report["sheet_column"] = names[1] if len(names) > 1 else None + self._check_width() + + def _claim_provenance_name(self, base, taken, code, label): + candidate = base + suffix = 2 + while candidate.casefold() in taken: + candidate = _suffixed_name(base, suffix) + suffix += 1 + if candidate != base: + self.report["warnings"].append({ + "code": code, + "message": f'The files already have a "{base}" column, so the {label} is "{candidate}".', + }) + return candidate + + def _unique_data_name(self, display): + taken = {name.casefold() for name in (self.provenance or []) + self.data_display} + if display.casefold() not in taken: + return display + suffix = 2 + candidate = _suffixed_name(display, suffix) + while candidate.casefold() in taken: + suffix += 1 + candidate = _suffixed_name(display, suffix) + self.report["warnings"].append({ + "code": "column_renamed", + "message": f'A column named "{display}" is shown as "{candidate}" because that name is already used.', + }) + return candidate + + def _check_width(self): + width = len(self.provenance or ()) + len(self.data_keys) + if width > self.limits.max_columns: + raise TabularMergeError( + "too_many_columns", + f"The merged table would have {width} columns; at most {self.limits.max_columns} are supported.", + ) + + def _require_named_columns(self, keys, code, purpose): + missing = [key for key in keys if key not in self.data_index] + if missing: + raise TabularMergeError( + code, f"The column(s) to {purpose} by aren't in the merged table: {', '.join(_quoted(missing))}.", + ) + + # -- tables ------------------------------------------------------------ + + def _add_table(self, source, table, entry): + header = _header_names(table.header, source, self.limits, entry) + header = _apply_aliases(header, self.alias_index, source, entry) + policy = self.options.schema_policy + if policy in (SCHEMA_POLICY_BY_NAME, SCHEMA_POLICY_EXACT_ORDER): + if self.reference is None: + self.reference = header + for display, key in header: + self._add_data_column(display, key) + self._init_provenance(set(self.data_keys)) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + mapping = list(range(len(header))) + else: + mapping = _match_header(self.reference, header, policy, entry) + if mapping is None: + self._incompatible(entry, "columns_differ") + return + elif policy == SCHEMA_POLICY_UNION: + mapping = self._union_mapping(header, entry) + else: + mapping = self._mapped_mapping(header, entry) + if mapping is None: + self._incompatible(entry, "no_matching_columns") + return + + if self.mismatch: + entry["status"] = SOURCE_STATUS_NOT_PROCESSED + return + self._append_rows(source, table, entry, mapping, len(header)) + entry["status"] = SOURCE_STATUS_MERGED + self.merged_tables += 1 + for position, column in enumerate(mapping): + if column is not None: + self.seen_counts[position] += 1 + + def _union_mapping(self, header, entry): + first = self.provenance is None + if first: + self._init_provenance({key for _, key in header}) + added = [] + for display, key in header: + if key not in self.data_index: + name = self._unique_data_name(display) + self._add_data_column(name, key) + self._check_width() + added.append(name) + positions = {key: index for index, (_, key) in enumerate(header)} + mapping = [positions.get(key) for key in self.data_keys] + if not first: + entry["missing_columns"] = [ + name for name, column in zip(self.data_display, mapping) if column is None + ] + entry["extra_columns"] = added + return mapping + + def _mapped_mapping(self, header, entry): + positions = {key: index for index, (_, key) in enumerate(header)} + mapping = [positions.get(key) for key in self.data_keys] + entry["missing_columns"] = [name for name, column in zip(self.data_display, mapping) if column is None] + entry["ignored_columns"] = [display for display, key in header if key not in self.data_index] + if all(column is None for column in mapping): + return None + return mapping + + def _incompatible(self, entry, reason): + entry["reason"] = reason + if self.options.on_incompatible == ON_INCOMPATIBLE_EXCLUDE: + entry["status"] = SOURCE_STATUS_EXCLUDED + self.excluded_tables += 1 + else: + entry["status"] = SOURCE_STATUS_SCHEMA_MISMATCH + self.mismatch = True + + # -- rows -------------------------------------------------------------- + + def _append_rows(self, source, table, entry, mapping, width): + options = self.options + limits = self.limits + prefix = [] + if self.provenance: + prefix.append(source.file_name) + if len(self.provenance) > 1: + prefix.append(table.sheet) + if table.sheet is None: + self.sheet_null = True + self.row_starts.append(self.spool.count) + self.row_entries.append(entry) + dedupe_positions = ( + [self.data_index.get(key) for key in self.dedupe_keys] + if options.dedupe == DEDUPE_KEY_COLUMNS else None + ) + sort_positions = [self.data_index.get(key) for key in self.sort_keys] + keep_last = options.dedupe != DEDUPE_NONE and options.dedupe_keep == DEDUPE_KEEP_LAST + checked = 0 + for row_number, values in table.rows(): + if checked % _CANCEL_CHECK_INTERVAL == 0: + _check_cancel(self.cancel_requested) + checked += 1 + cells = list(values) + if len(cells) > width: + if any(_cell_has_content(value) for value in cells[width:]): + raise TabularMergeError( + "row_has_extra_values", + f"Row {row_number} in {_safe_name(source)} has more values than there are columns.", + ) + cells = cells[:width] + if not any(_cell_has_content(value) for value in cells): + entry["blank_rows_skipped"] += 1 + self.blank_rows += 1 + continue + if len(cells) < width: + entry["short_rows_padded"] += 1 + cells.extend([""] * (width - len(cells))) + for value in cells: + if len(value) > limits.max_cell_chars: + raise TabularMergeError( + "cell_too_large", + f"Row {row_number} in {_safe_name(source)} has a value longer than " + f"{limits.max_cell_chars} characters.", + ) + if self.rows_read >= limits.max_total_rows: + raise TabularMergeError( + "row_limit_exceeded", + f"The selected files hold more than {limits.max_total_rows:,} rows. " + "Merge fewer files here, or use a workflow for larger merges.", + ) + self.rows_read += 1 + data = [cells[column] if column is not None else None for column in mapping] + if options.dedupe != DEDUPE_NONE: + digest = _row_digest( + data if dedupe_positions is None else _values_at(data, dedupe_positions) + ) + if digest is not None: + if keep_last: + previous = self.last_digest.get(digest) + if previous is not None: + owner = self._entry_for_row(previous) + owner["rows"] -= 1 + owner["duplicates_removed"] += 1 + self.duplicates_removed += 1 + self.last_digest[digest] = self.spool.count + elif digest in self.seen_digests: + entry["duplicates_removed"] += 1 + self.duplicates_removed += 1 + continue + else: + self.seen_digests.add(digest) + if self.sort_keys: + if self.spool.count >= limits.max_sort_rows: + raise TabularMergeError( + "sort_limit_exceeded", + f"Sorting is available for merges of up to {limits.max_sort_rows:,} rows.", + ) + self.sort_values.append(tuple( + _sort_value(data[column] if column is not None else None, item) + for column, item in zip(sort_positions, options.sort_by) + )) + self.spool.append(prefix + data) + entry["rows"] += 1 + + def _entry_for_row(self, index): + return self.row_entries[bisect.bisect_right(self.row_starts, index) - 1] + + # -- finish ------------------------------------------------------------ + + def finish(self): + report = self.report + if self.mismatch: + report["status"] = "failed" + report["columns"] = list(self.provenance or ()) + list(self.data_display) + for entry in report["sources"]: + if entry["status"] in (SOURCE_STATUS_MERGED, SOURCE_STATUS_NOT_PROCESSED): + entry["status"] = SOURCE_STATUS_COMPATIBLE + entry["rows"] = 0 + entry["blank_rows_skipped"] = 0 + entry["short_rows_padded"] = 0 + entry["duplicates_removed"] = 0 + raise TabularMergeError("schema_mismatch", _mismatch_message(report), report=report) + if self.merged_tables == 0: + report["status"] = "failed" + raise TabularMergeError( + "nothing_to_merge", + "None of the selected files has a sheet with column headers and matching columns to merge.", + report=report, + ) + self._require_named_columns(self.dedupe_keys, "dedupe_column_not_found", "remove duplicates") + self._require_named_columns(self.sort_keys, "sort_column_not_found", "sort") + + columns = tuple(self.provenance) + tuple(self.data_display) + nullable = set() + if len(self.provenance) > 1 and self.sheet_null: + nullable.add(self.provenance[1]) + for name, seen in zip(self.data_display, self.seen_counts): + if seen < self.merged_tables: + nullable.add(name) + keep_last = self.options.dedupe != DEDUPE_NONE and self.options.dedupe_keep == DEDUPE_KEEP_LAST + row_count = self.spool.count - (self.duplicates_removed if keep_last else 0) + order = self._sort_order() if self.sort_keys else None + keep = self._keep_last_predicate() if keep_last else None + + report["status"] = "merged" + report["columns"] = list(columns) + report["nullable_columns"] = [name for name in columns if name in nullable] + report["totals"] = { + "sources": report["totals"]["sources"], + "tables_merged": self.merged_tables, + "rows": row_count, + "columns": len(columns), + "blank_rows_skipped": self.blank_rows, + "duplicates_removed": self.duplicates_removed, + "excluded": self.excluded_tables, + } + limitations = [] + if any(entry.get("format") != TABULAR_MERGE_FORMAT_CSV for entry in report["sources"]): + limitations.extend(_EXCEL_LIMITATIONS) + if self.excluded_tables: + considered = sum( + entry["status"] in (SOURCE_STATUS_MERGED, SOURCE_STATUS_EXCLUDED) for entry in report["sources"] + ) + limitations.append(_EXCLUDED_LIMITATION.format(count=self.excluded_tables, total=considered)) + if any(name in nullable for name in self.data_display): + limitations.append(_BLANK_COLUMNS_LIMITATION) + if any(entry.get("ignored_columns") for entry in report["sources"] if entry["status"] == SOURCE_STATUS_MERGED): + limitations.append(_IGNORED_COLUMNS_LIMITATION) + report["limitations"] = limitations + result = TabularMergeResult( + columns, report, self.spool, nullable=nullable, row_count=row_count, order=order, keep=keep, + ) + self.sort_values = [] + return result + + def _sort_order(self): + values = self.sort_values + order = list(range(self.spool.count)) + for position in reversed(range(len(self.options.sort_by))): + descending = self.options.sort_by[position].descending + order.sort(key=lambda index, position=position: values[index][position], reverse=descending) + return order + + def _keep_last_predicate(self): + offset = len(self.provenance) + positions = ( + [self.data_index[key] for key in self.dedupe_keys] + if self.options.dedupe == DEDUPE_KEY_COLUMNS else None + ) + last = self.last_digest + + def keep(index, values): + data = values[offset:] + digest = _row_digest(data if positions is None else _values_at(data, positions)) + return digest is None or last.get(digest) == index + + return keep + + +def _validate_sources(sources, limits, *, minimum=2): + if len(sources) < minimum: + raise TabularMergeError( + "too_few_sources", "Merging needs at least two files." if minimum > 1 else "Select a file to inspect.", + ) + if len(sources) > limits.max_sources: + raise TabularMergeError( + "too_many_sources", + f"{len(sources)} files were selected; at most {limits.max_sources} can be used here.", + ) + seen = set() + for source in sources: + if not isinstance(source, TabularMergeSource): + raise TabularMergeError("invalid_source", "Each merge source must be a TabularMergeSource.") + if not isinstance(source.source_id, str) or not source.source_id.strip(): + raise TabularMergeError("invalid_source", "Each merge source needs an identifier.") + if source.source_id in seen: + raise TabularMergeError("duplicate_source", "The same file was selected more than once.") + seen.add(source.source_id) + if not callable(source.load_bytes): + raise TabularMergeError("invalid_source", "Each merge source needs a byte loader.") + if TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()) is None: + raise TabularMergeError( + "unsupported_format", + f"{_safe_name(source)} isn't a CSV or Excel file (.csv, .xlsx, .xlsm, .xls).", + ) + + +def _new_report(options, source_count): + return { + "version": TABULAR_MERGE_REPORT_VERSION, + "status": "running", + "policy": options.policy_report(), + "columns": [], + "nullable_columns": [], + "source_column": None, + "sheet_column": None, + "totals": { + "sources": source_count, "tables_merged": 0, "rows": 0, "columns": 0, + "blank_rows_skipped": 0, "duplicates_removed": 0, "excluded": 0, + }, + "sources": [], + "warnings": [], + "limitations": [], + } + + +def _new_source_entry(source, reader, sheet): + entry = { + "source_id": source.source_id, + "file_name": _safe_name(source), + "format": TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()), + "sheet": sheet, + "encoding": None, + "delimiter": None, + "header_row": None, + "rows": 0, + "blank_rows_skipped": 0, + "short_rows_padded": 0, + "duplicates_removed": 0, + "status": SOURCE_STATUS_NOT_PROCESSED, + "reason": None, + "missing_columns": [], + "extra_columns": [], + "ignored_columns": [], + "renamed_columns": [], + "order_differs": False, + "warnings": [], + } + entry.update(reader.describe()) + return entry + + +def _safe_name(source): + name = str(getattr(source, "file_name", "") or "").strip() + return name or "A selected file" + + +def _suffixed_name(base, suffix): + ending = f" ({suffix})" + name = base + while len((name + ending).encode("utf-8")) > MAX_COLUMN_NAME_BYTES and name: + name = name[:-1].rstrip() + return name + ending + + +def _check_cancel(cancel_requested): + if cancel_requested is not None and cancel_requested(): + raise TabularMergeCancelled() + + +def _load_source_bytes(source, limits): + content = source.load_bytes() + if isinstance(content, bytearray): + content = bytes(content) + if not isinstance(content, bytes): + raise TabularMergeError("source_unavailable", f"{_safe_name(source)} could not be read.") + if len(content) > limits.max_source_bytes: + raise TabularMergeError( + "source_too_large", + f"{_safe_name(source)} is larger than the {limits.max_source_bytes // (1024 * 1024)} MB merge limit.", + ) + return content + + +def _alias_index(options): + index = {} + for target, aliases in options.column_aliases: + target_key = normalize_header_name(target) + for alias in aliases: + index[normalize_header_name(alias)] = (target, target_key) + return index + + +def _header_names(raw_header, source, limits, entry): + """Display and comparison names for one header row; trailing blanks are dropped.""" + cells = list(raw_header) + while cells and not _display_header(cells[-1]): + cells.pop() + if not cells: + raise TabularMergeError("empty_source", f"{_safe_name(source)} has no header row.") + names = [] + seen = {} + for position, value in enumerate(cells, start=1): + display = _display_header(value) + if not display: + display = f"Column {position}" + entry["warnings"].append({ + "code": "blank_header", + "message": f'Column {position} has no header, so it is named "{display}".', + }) + if len(display.encode("utf-8")) > limits.max_header_bytes: + raise TabularMergeError( + "header_too_long", + f"A column header in {_safe_name(source)} is longer than {limits.max_header_bytes} bytes.", + ) + normalized = display.casefold() + if normalized in seen: + raise TabularMergeError( + "duplicate_columns", + f'{_safe_name(source)} has more than one "{display}" column.', + ) + seen[normalized] = position + names.append((display, normalized)) + return names + + +def _apply_aliases(header, alias_index, source, entry): + """Rename headers that are aliases of a target column; renaming must not duplicate one.""" + if not alias_index: + return header + renamed = [] + result = [] + seen = set() + for display, key in header: + target = alias_index.get(key) + if target is not None: + renamed.append({"from": display, "to": target[0]}) + display, key = target + if key in seen: + raise TabularMergeError( + "duplicate_columns", + f'{_safe_name(source)} has more than one column that becomes "{display}".', + ) + seen.add(key) + result.append((display, key)) + entry["renamed_columns"] = renamed + return result + + +def _match_header(reference, header, schema_policy, entry): + """Positions in ``header`` for each reference column, or None on a mismatch.""" + reference_keys = [normalized for _, normalized in reference] + header_keys = [normalized for _, normalized in header] + positions = {key: index for index, key in enumerate(header_keys)} + missing = [display for display, key in reference if key not in positions] + reference_set = set(reference_keys) + extra = [display for display, key in header if key not in reference_set] + entry["missing_columns"] = missing + entry["extra_columns"] = extra + if missing or extra: + return None + if reference_keys != header_keys: + entry["order_differs"] = True + if schema_policy == SCHEMA_POLICY_EXACT_ORDER: + return None + return [positions[key] for key in reference_keys] + + +def _mismatch_message(report): + reference = report["sources"][0]["file_name"] if report["sources"] else "the first file" + parts = [] + for entry in report["sources"]: + if entry["status"] != SOURCE_STATUS_SCHEMA_MISMATCH: + continue + details = [] + if entry["reason"] == "no_matching_columns": + details.append("none of the requested columns") + if entry["missing_columns"] and entry["reason"] != "no_matching_columns": + details.append(f"missing {', '.join(_quoted(entry['missing_columns']))}") + if entry["extra_columns"]: + details.append(f"extra {', '.join(_quoted(entry['extra_columns']))}") + if not details and entry["order_differs"]: + details.append("columns in a different order") + label = entry["file_name"] if not entry.get("sheet") else f'{entry["file_name"]} sheet "{entry["sheet"]}"' + parts.append(f"{label} ({'; '.join(details)})") + return ( + f"These files don't have the same columns as {reference}: {'; '.join(parts)}. " + "Nothing was merged." + ) + + +def _quoted(values, limit=8): + values = list(values) + shown = [f'"{value}"' for value in values[:limit]] + if len(values) > limit: + shown.append(f"and {len(values) - limit} more") + return shown + + +def _cell_has_content(value): + return bool(value) and bool(str(value).strip()) + + +def _values_at(data, positions): + return [data[position] if position is not None and position < len(data) else None for position in positions] + + +def _row_digest(values): + """A collision-resistant key for duplicate detection, or None when every value is blank. + + Missing (null) and empty values compare equal, and trailing blanks are ignored, so a row + read before a later file added a column matches the same row read after it. + """ + normalized = ["" if value is None else value for value in values] + while normalized and normalized[-1] == "": + normalized.pop() + if not any(_cell_has_content(value) for value in normalized): + return None + encoded = json.dumps(normalized, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return hashlib.blake2b(encoded, digest_size=16).digest() + + +def _sort_value(value, sort): + """A comparable key; typed values sort before text that isn't that type, blanks last.""" + ascending = not sort.descending + typed_rank = 0 if ascending else 2 + blank_rank = 2 if ascending else 0 + if value is None or not str(value).strip(): + return (blank_rank, "") + if sort.value_type == SORT_TYPE_NUMBER: + number = _parse_number(value) + if number is not None: + return (typed_rank, number) + return (1, value.casefold()) + if sort.value_type == SORT_TYPE_DATE: + moment = _parse_date(value) + if moment is not None: + return (typed_rank, moment) + return (1, value.casefold()) + return (typed_rank, value.casefold()) + + +def _parse_number(value): + text = value.strip() + if text[:1] in _CURRENCY_PREFIXES: + text = text[1:].strip() + if not _NUMBER_TEXT.fullmatch(text): + return None + try: + number = float(text.replace(",", "")) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _parse_date(value): + text = value.strip() + if len(text) < 10 or not text[:4].isdigit(): + return None + try: + if len(text) == 10: + day = date.fromisoformat(text) + return datetime(day.year, day.month, day.day) + moment = datetime.fromisoformat(text) + except ValueError: + return None + if moment.tzinfo is not None: + moment = moment.astimezone(timezone.utc).replace(tzinfo=None) + return moment + + +# --------------------------------------------------------------------------- +# Inspection +# --------------------------------------------------------------------------- + + +def inspect_tabular_sources( + sources: Sequence[TabularMergeSource], + *, + options: Optional[TabularInspectOptions] = None, + limits: Optional[TabularMergeLimits] = None, + cancel_requested: Optional[Callable[[], bool]] = None, +) -> dict: + """Describe each file's sheets, headers, row counts and samples, and how their columns line up. + + A file that can't be read is reported with its problem rather than failing the inspection. + """ + options = options or TabularInspectOptions() + limits = limits or TabularMergeLimits() + sources = list(sources or ()) + _validate_sources(sources, limits, minimum=1) + report = { + "version": TABULAR_INSPECTION_VERSION, + "options": { + "sheet": options.sheet, "sheets": options.sheets, + "header_row": options.header_row, "sample_rows": options.sample_rows, + }, + "sources": [], + "compatibility": None, + "samples_omitted": False, + "limitations": [], + } + inspected = [] + for source in sources: + _check_cancel(cancel_requested) + entry = { + "source_id": source.source_id, + "file_name": _safe_name(source), + "format": TABULAR_MERGE_EXTENSION_FORMATS.get(source.resolved_extension()), + "status": "inspected", + "problem": None, + "encoding": None, + "delimiter": None, + "sheets": [], + "tables": [], + } + report["sources"].append(entry) + reader = None + try: + content = _load_source_bytes(source, limits) + reader = _open_reader(source, content, limits) + entry.update(reader.describe()) + entry["sheets"] = [{"name": name, "visible": state == "visible"} for name, state in reader.sheet_states()] + for sheet, skip_reason in reader.plan(options, limits): + _check_cancel(cancel_requested) + table_entry = {"sheet": sheet, "status": "inspected", "reason": None, "problem": None} + entry["tables"].append(table_entry) + if skip_reason is not None: + table_entry.update(status=SOURCE_STATUS_SKIPPED, reason=skip_reason) + continue + try: + table = reader.open_table(sheet, options.header_row) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + table_entry.update(status="problem", problem={"code": exc.code, "message": exc.message}) + continue + try: + header = _inspect_table(source, table, table_entry, options, limits, cancel_requested) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + table_entry.update(status="problem", problem={"code": exc.code, "message": exc.message}) + continue + finally: + table.close() + inspected.append((entry, table_entry, header)) + except TabularMergeCancelled: + raise + except TabularMergeError as exc: + entry["status"] = "problem" + entry["problem"] = {"code": exc.code, "message": exc.message} + finally: + if reader is not None: + reader.close() + + report["compatibility"] = _compatibility(inspected) + limitations = ["Row counts leave out blank rows."] + if options.sample_rows: + limitations.append( + f"Samples show up to {options.sample_rows} row(s) per sheet, and sample values longer than " + f"{_INSPECT_CELL_CHARS} characters are shortened." + ) + if any(entry["format"] != TABULAR_MERGE_FORMAT_CSV for entry in report["sources"]): + limitations.append("Excel formulas are shown as their last calculated values.") + report["limitations"] = limitations + _fit_inspection_budget(report) + return report + + +def _inspect_table(source, table, table_entry, options, limits, cancel_requested): + probe = {"warnings": []} + header = _header_names(table.header, source, limits, probe) + width = len(header) + samples = [] + count = 0 + capped = False + extra_rows = 0 + first_data_row = None + for row_number, values in table.rows(): + if count % _CANCEL_CHECK_INTERVAL == 0: + _check_cancel(cancel_requested) + cells = list(values) + if not any(_cell_has_content(value) for value in cells): + continue + if first_data_row is None: + first_data_row = (row_number, cells) + if len(cells) > width and any(_cell_has_content(value) for value in cells[width:]): + extra_rows += 1 + count += 1 + if len(samples) < options.sample_rows: + row = [_clip_sample(value) for value in cells[:width]] + samples.append(row + [""] * (width - len(row))) + if count >= limits.max_total_rows: + capped = True + break + warnings = list(probe["warnings"]) + header_values = sum(_cell_has_content(value) for value in table.header) + if ( + options.header_row is None and header_values == 1 and first_data_row is not None + and sum(_cell_has_content(value) for value in first_data_row[1]) > 1 + ): + warnings.append({ + "code": "possible_title_row", + "message": ( + f"Row {table.header_row} has only one value, so it may be a title rather than column " + f"headers; row {first_data_row[0]} may hold the headers." + ), + "suggested_header_row": first_data_row[0], + }) + if extra_rows: + warnings.append({ + "code": "rows_with_extra_values", + "message": f"{extra_rows} row(s) have more values than there are column headers.", + }) + table_entry.update({ + "header_row": table.header_row, + "columns": [display for display, _ in header], + "row_count": count, + "row_count_capped": capped, + "sample_rows": samples, + "warnings": warnings, + }) + return header + + +def _clip_sample(value): + text = "" if value is None else str(value) + if len(text) > _INSPECT_CELL_CHARS: + return text[:_INSPECT_CELL_CHARS - 1] + "\u2026" + return text + + +def _compatibility(inspected): + groups = {} + order = None + same_order = True + presence = {} + for entry, table_entry, header in inspected: + keys = [key for _, key in header] + group = groups.setdefault( + tuple(sorted(keys)), {"columns": [display for display, _ in header], "tables": []}, + ) + group["tables"].append({ + "source_id": entry["source_id"], "file_name": entry["file_name"], "sheet": table_entry["sheet"], + }) + if order is None: + order = keys + elif keys != order: + same_order = False + for display, key in header: + item = presence.setdefault(key, {"name": display, "present_in": 0}) + item["present_in"] += 1 + near = {} + for key, item in presence.items(): + squashed = _NOT_WORD.sub("", key) + if squashed: + near.setdefault(squashed, []).append(item["name"]) + similar = [{"names": names} for names in near.values() if len(names) > 1] + identical = len(groups) == 1 + if not inspected: + suggested = None + elif identical: + suggested = SCHEMA_POLICY_BY_NAME + else: + suggested = SCHEMA_POLICY_MAPPED if similar else SCHEMA_POLICY_UNION + return { + "tables": len(inspected), + "identical_columns": bool(inspected) and identical, + "same_order": bool(inspected) and identical and same_order, + "groups": list(groups.values()), + "columns": [{"name": item["name"], "present_in": item["present_in"]} for item in presence.values()], + "similar_columns": similar, + "suggested_policy": suggested, + } + + +def _json_size(value): + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + + +def _fit_inspection_budget(report): + if _json_size(report) <= _INSPECT_MAX_BYTES: + return + for entry in report["sources"]: + for table in entry["tables"]: + if table.get("sample_rows"): + table["sample_rows"] = [] + report["samples_omitted"] = True + report["limitations"].append("Sample rows were left out to keep the inspection small.") + if _json_size(report) > _INSPECT_MAX_BYTES: + raise TabularMergeError( + "inspection_too_large", "The selected files have too many sheets or columns to inspect here.", + ) + + +# --------------------------------------------------------------------------- +# Readers +# --------------------------------------------------------------------------- + + +class _Table: + """A header plus a row iterator over one sheet or CSV body.""" + + format = "" + sheet = None + header_row = None + + @property + def header(self): + raise NotImplementedError + + def rows(self): + raise NotImplementedError + + def close(self): + return None + + +class _Reader: + """One opened source file; workbooks can hold several tables.""" + + format = "" + + def __init__(self, source): + self.source = source + + def describe(self): + return {} + + def sheet_states(self): + return [] + + def plan(self, options, limits): + """``(sheet, skip_reason)`` for each table to read, in order.""" + states = self.sheet_states() + if options.sheets == SHEETS_ALL: + if not states: + raise TabularMergeError("empty_source", f"{_safe_name(self.source)} has no worksheets.") + if len(states) > limits.max_sheets_per_source: + raise TabularMergeError( + "too_many_sheets", + f"{_safe_name(self.source)} has more than {limits.max_sheets_per_source} sheets.", + ) + return [(name, None if state == "visible" else "hidden_sheet") for name, state in states] + return [(_select_sheet(states, options.sheet, self.source), None)] + + def open_table(self, sheet, header_row): + raise NotImplementedError + + def close(self): + return None + + +def _open_reader(source, content, limits): + table_format = TABULAR_MERGE_EXTENSION_FORMATS[source.resolved_extension()] + if table_format == TABULAR_MERGE_FORMAT_CSV: + return _CsvReader(source, content) + if table_format == TABULAR_MERGE_FORMAT_XLSX: + return _XlsxReader(source, content, limits) + return _XlsReader(source, content) + + +class _CsvReader(_Reader): + format = TABULAR_MERGE_FORMAT_CSV + + def __init__(self, source, content): + super().__init__(source) + self._text, self._encoding = _decode_csv(content, source) + self._delimiter = _detect_delimiter(self._text) + + def describe(self): + return {"encoding": self._encoding, "delimiter": self._delimiter} + + def plan(self, options, limits): + return [(None, None)] + + def open_table(self, sheet, header_row): + return _CsvTable(self.source, self._text, self._delimiter, header_row) + + +class _CsvTable(_Table): + format = TABULAR_MERGE_FORMAT_CSV + + def __init__(self, source, text, delimiter, header_row): + self._source = source + self._reader = csv.reader( + io.StringIO(text, newline=""), delimiter=delimiter, quotechar='"', doublequote=True, strict=False, + ) + self._header = None + # Rows are counted as records, as a spreadsheet shows them, not as physical lines. + record = 0 + try: + for values in self._reader: + record += 1 + if header_row is not None and record < header_row: + continue + if any(_cell_has_content(value) for value in values): + self._header = values + self.header_row = record + self._record = record + break + if header_row is not None: + raise TabularMergeError( + "empty_source", f"Row {header_row} in {_safe_name(source)} is blank, so it has no column headers.", + ) + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} couldn't be read as CSV near line {self._reader.line_num}.", + ) from exc + if self._header is None: + raise TabularMergeError( + "empty_source", + f"{_safe_name(source)} has no header row." if header_row is None + else f"{_safe_name(source)} has fewer than {header_row} rows.", + ) + + @property + def header(self): + return self._header + + def rows(self): + reader = self._reader + try: + for values in reader: + self._record += 1 + yield self._record, values + except csv.Error as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(self._source)} couldn't be read as CSV near line {reader.line_num}.", + ) from exc + + +def _decode_csv(content, source): + for bom, encoding in ( + (codecs.BOM_UTF32_LE, "utf-32"), (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF8, "utf-8-sig"), + (codecs.BOM_UTF16_LE, "utf-16"), (codecs.BOM_UTF16_BE, "utf-16"), + ): + if content.startswith(bom): + try: + return content.decode(encoding), encoding + except UnicodeDecodeError as exc: + raise TabularMergeError( + "unreadable_csv", f"{_safe_name(source)} has text that doesn't match its encoding.", + ) from exc + if b"\x00" in content[:_CSV_SNIFF_BYTES]: + raise TabularMergeError("unreadable_csv", f"{_safe_name(source)} doesn't look like a text CSV file.") + for encoding in ("utf-8", "cp1252"): + try: + return content.decode(encoding), encoding + except UnicodeDecodeError: + continue + return content.decode("latin-1"), "latin-1" + + +def _detect_delimiter(text): + """The most frequent supported delimiter outside quotes on the first nonblank line.""" + counts = {delimiter: 0 for delimiter in _CSV_DELIMITERS} + in_quotes = False + seen_content = False + for character in text[:_CSV_SNIFF_BYTES]: + if character == '"': + in_quotes = not in_quotes + seen_content = True + continue + if in_quotes: + continue + if character in "\r\n": + if seen_content: + break + continue + if character in counts: + counts[character] += 1 + if not character.isspace(): + seen_content = True + best = max(_CSV_DELIMITERS, key=lambda delimiter: (counts[delimiter], -_CSV_DELIMITERS.index(delimiter))) + return best if counts[best] else "," + + +class _XlsxReader(_Reader): + format = TABULAR_MERGE_FORMAT_XLSX + + def __init__(self, source, content, limits): + super().__init__(source) + _guard_workbook_archive(content, source, limits) + # openpyxl is imported only when a workbook is merged; it is a large binary dependency. + from openpyxl import load_workbook + + try: + self._workbook = load_workbook( + io.BytesIO(content), read_only=True, data_only=True, keep_links=False, + ) + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} couldn't be opened as an Excel workbook.", + ) from exc + + def sheet_states(self): + try: + return [ + (worksheet.title, getattr(worksheet, "sheet_state", "visible")) + for worksheet in self._workbook.worksheets + ] + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def open_table(self, sheet, header_row): + try: + return _XlsxTable(self.source, self._workbook[sheet], sheet, header_row) + except TabularMergeError: + raise + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def close(self): + workbook = getattr(self, "_workbook", None) + if workbook is not None: + self._workbook = None + try: + workbook.close() + except Exception: + # The reader is closing; library cleanup failures are safe to ignore. + pass + + +class _XlsxTable(_Table): + format = TABULAR_MERGE_FORMAT_XLSX + + def __init__(self, source, worksheet, sheet, header_row): + self._source = source + self.sheet = sheet + worksheet.reset_dimensions() + self._iterator = worksheet.iter_rows(values_only=True) + self._row_number = 0 + self._header = None + for values in self._iterator: + self._row_number += 1 + if header_row is not None and self._row_number < header_row: + continue + cells = [_excel_text(value) for value in values] + if any(_cell_has_content(value) for value in cells): + self._header = cells + self.header_row = self._row_number + break + if header_row is not None: + raise TabularMergeError( + "empty_source", + f'Row {header_row} of sheet "{sheet}" in {_safe_name(source)} is blank, so it has no column headers.', + ) + if self._header is None: + raise TabularMergeError( + "empty_source", f'Sheet "{sheet}" in {_safe_name(source)} has no header row.', + ) + + @property + def header(self): + return self._header + + def rows(self): + try: + for values in self._iterator: + self._row_number += 1 + yield self._row_number, [_excel_text(value) for value in values] + except TabularMergeError: + raise + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self._source)} couldn't be read near row {self._row_number}.", + ) from exc + + +def _guard_workbook_archive(content, source, limits): + if content[:8] == _OLE_MAGIC: + raise TabularMergeError( + "encrypted_workbook", + f"{_safe_name(source)} is password-protected or encrypted, so it can't be merged.", + ) + if content[:4] != _ZIP_MAGIC: + raise TabularMergeError("unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel workbook.") + try: + archive = zipfile.ZipFile(io.BytesIO(content)) + except zipfile.BadZipFile as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel workbook.", + ) from exc + with archive: + entries = archive.infolist() + if len(entries) > limits.max_workbook_entries: + raise TabularMergeError("workbook_too_large", f"{_safe_name(source)} has too many internal parts to merge.") + if sum(entry.file_size for entry in entries) > limits.max_workbook_uncompressed_bytes: + raise TabularMergeError( + "workbook_too_large", f"{_safe_name(source)} is too large to merge once uncompressed.", + ) + try: + unsafe = first_unsafe_xml_part(archive, entries) + except UnreadablePackageError as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel workbook.", + ) from exc + if unsafe is not None: + raise TabularMergeError( + "unreadable_workbook", + f"{_safe_name(source)} isn't a valid Excel workbook; it has XML that Excel files can't contain.", + ) + + +class _XlsReader(_Reader): + format = TABULAR_MERGE_FORMAT_XLS + + def __init__(self, source, content): + super().__init__(source) + if content[:8] != _OLE_MAGIC: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(source)} isn't a valid Excel 97-2003 (.xls) workbook.", + ) + # xlrd is imported only when a legacy workbook is merged. + import xlrd + + self._xlrd = xlrd + try: + self._book = xlrd.open_workbook(file_contents=content, on_demand=True) + except Exception as exc: + code = "encrypted_workbook" if "encrypt" in str(exc).lower() else "unreadable_workbook" + message = ( + f"{_safe_name(source)} is password-protected or encrypted, so it can't be merged." + if code == "encrypted_workbook" + else f"{_safe_name(source)} couldn't be opened as an Excel workbook." + ) + raise TabularMergeError(code, message) from exc + + def sheet_states(self): + try: + names = self._book.sheet_names() + visibility = getattr(self._book, "_sheet_visibility", None) + if not isinstance(visibility, list) or len(visibility) != len(names): + visibility = [ + getattr(self._book.sheet_by_index(index), "visibility", 0) for index in range(len(names)) + ] + return [(name, "visible" if state == 0 else "hidden") for name, state in zip(names, visibility)] + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def open_table(self, sheet, header_row): + try: + worksheet = self._book.sheet_by_name(sheet) + return _XlsTable(self.source, self._xlrd, worksheet, self._book.datemode, sheet, header_row) + except TabularMergeError: + raise + except Exception as exc: + raise TabularMergeError( + "unreadable_workbook", f"{_safe_name(self.source)} couldn't be read as an Excel workbook.", + ) from exc + + def close(self): + book = getattr(self, "_book", None) + if book is not None: + self._book = None + try: + book.release_resources() + except Exception: + # The reader is closing; library cleanup failures are safe to ignore. + pass + + +class _XlsTable(_Table): + format = TABULAR_MERGE_FORMAT_XLS + + def __init__(self, source, xlrd, worksheet, datemode, sheet, header_row): + self._source = source + self._xlrd = xlrd + self._worksheet = worksheet + self._datemode = datemode + self.sheet = sheet + self._next_row = 0 + self._header = None + while self._next_row < worksheet.nrows: + row_number = self._next_row + 1 + cells = self._row_values(self._next_row) + self._next_row += 1 + if header_row is not None and row_number < header_row: + continue + if any(_cell_has_content(value) for value in cells): + self._header = cells + self.header_row = row_number + break + if header_row is not None: + raise TabularMergeError( + "empty_source", + f'Row {header_row} of sheet "{sheet}" in {_safe_name(source)} is blank, so it has no column headers.', + ) + if self._header is None: + raise TabularMergeError( + "empty_source", f'Sheet "{sheet}" in {_safe_name(source)} has no header row.', + ) + + @property + def header(self): + return self._header + + def rows(self): + while self._next_row < self._worksheet.nrows: + row_number = self._next_row + 1 + cells = self._row_values(self._next_row) + self._next_row += 1 + yield row_number, cells + + def _row_values(self, row_index): + xlrd = self._xlrd + values = [] + for cell in self._worksheet.row(row_index): + if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK): + values.append("") + elif cell.ctype == xlrd.XL_CELL_DATE: + try: + values.append(_excel_text(xlrd.xldate_as_datetime(cell.value, self._datemode))) + except Exception: + values.append(_excel_text(cell.value)) + elif cell.ctype == xlrd.XL_CELL_BOOLEAN: + values.append("TRUE" if cell.value else "FALSE") + elif cell.ctype == xlrd.XL_CELL_ERROR: + values.append(xlrd.error_text_from_code.get(cell.value, "#ERROR")) + else: + values.append(_excel_text(cell.value)) + return values + + +def _select_sheet(sheets: List[Tuple[str, str]], requested, source): + names = [name for name, _ in sheets] + if not names: + raise TabularMergeError("empty_source", f"{_safe_name(source)} has no worksheets.") + if requested is not None: + if requested in names: + return requested + folded = requested.casefold() + for name in names: + if name.casefold() == folded: + return name + raise TabularMergeError( + "sheet_not_found", + f'{_safe_name(source)} has no sheet named "{requested}". Its sheets are: ' + f"{', '.join(_quoted(names))}.", + details={"available_sheets": names}, + ) + for name, state in sheets: + if state == "visible": + return name + return names[0] + + +def _excel_text(value): + """Canonical text for one Excel cell value.""" + if value is None: + return "" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + if value.is_integer() and abs(value) < _MAX_EXACT_INTEGER_FLOAT: + return str(int(value)) + return repr(value) + if isinstance(value, datetime): + if value.tzinfo is None and value.time() == time(0, 0): + return value.date().isoformat() + return value.isoformat(timespec="seconds" if value.microsecond == 0 else "microseconds") + if isinstance(value, date): + return value.isoformat() + if isinstance(value, time): + return value.isoformat(timespec="seconds" if value.microsecond == 0 else "microseconds") + if isinstance(value, timedelta): + total_seconds = int(value.total_seconds()) + sign = "-" if total_seconds < 0 else "" + hours, remainder = divmod(abs(total_seconds), 3600) + minutes, seconds = divmod(remainder, 60) + return f"{sign}{hours}:{minutes:02d}:{seconds:02d}" + return str(value) + + +__all__ = [ + "DEDUPE_EXACT_ROWS", + "DEDUPE_KEEP_FIRST", + "DEDUPE_KEEP_LAST", + "DEDUPE_KEY_COLUMNS", + "DEDUPE_NONE", + "DEFAULT_INSPECT_SAMPLE_ROWS", + "DEFAULT_SHEET_COLUMN_NAME", + "DEFAULT_SOURCE_COLUMN_NAME", + "MAX_INSPECT_SAMPLE_ROWS", + "ON_INCOMPATIBLE_EXCLUDE", + "ON_INCOMPATIBLE_FAIL", + "SCHEMA_POLICY_BY_NAME", + "SCHEMA_POLICY_EXACT_ORDER", + "SCHEMA_POLICY_MAPPED", + "SCHEMA_POLICY_UNION", + "SHEETS_ALL", + "SHEETS_FIRST", + "SORT_TYPE_DATE", + "SORT_TYPE_NUMBER", + "SORT_TYPE_TEXT", + "TABULAR_COLUMN_MAPPING_PROFILE", + "TABULAR_INSPECTION_VERSION", + "TABULAR_MERGE_DEDUPE_KEEP", + "TABULAR_MERGE_DEDUPE_MODES", + "TABULAR_MERGE_INCOMPATIBLE_MODES", + "TABULAR_MERGE_REPORT_VERSION", + "TABULAR_MERGE_SCHEMA_POLICIES", + "TABULAR_MERGE_SHEET_MODES", + "TABULAR_MERGE_SORT_TYPES", + "TABULAR_MERGE_SUPPORTED_EXTENSIONS", + "TabularInspectOptions", + "TabularMergeCancelled", + "TabularMergeError", + "TabularMergeLimits", + "TabularMergeOptions", + "TabularMergeResult", + "TabularMergeSort", + "TabularMergeSource", + "inspect_tabular_sources", + "iter_csv_rows", + "merge_tabular_sources", + "normalize_header_name", + "squash_header_name", + "tabular_column_mapping_schema", + "tabular_inspect_options_from_arguments", + "tabular_mapping_from_profile", + "tabular_merge_format_for_file_name", + "tabular_merge_options_from_arguments", +] diff --git a/application/single_app/functions_workflow_drafts.py b/application/single_app/functions_workflow_drafts.py index 3501b260e..c9dc62075 100644 --- a/application/single_app/functions_workflow_drafts.py +++ b/application/single_app/functions_workflow_drafts.py @@ -36,7 +36,17 @@ from content_screening.contracts import DocumentHeldError from functions_appinsights import log_event -from functions_document_actions import DOCUMENT_ACTION_TYPE_ANALYZE, get_enabled_document_action_types +from functions_document_actions import ( + DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, + MERGE_KIND_OPTION_KEYS, + MERGE_KIND_OUTPUT_FORMATS, + MERGE_KIND_TABULAR, + MERGE_KINDS_AVAILABLE, + get_enabled_document_action_types, + is_document_action_enabled, + normalize_document_action_config, +) from functions_file_sync import ( FILE_SYNC_SCOPE_GROUP, FILE_SYNC_SCOPE_PERSONAL, @@ -110,6 +120,9 @@ BLUEPRINT_RUN_AS = ('self', 'none') BLUEPRINT_SCOPE_TYPES = (FILE_SYNC_SCOPE_PERSONAL, FILE_SYNC_SCOPE_GROUP, FILE_SYNC_SCOPE_PUBLIC) BLUEPRINT_HANDLE_KINDS = ('documents', 'agents', 'sources') +# Where a merge task's files come from: its inputs, a sync, or the user's personal workspace. +BLUEPRINT_MERGE_FILES = ('inputs', 'changed', 'all', 'recent') +BLUEPRINT_MERGE_FILE_NAME_MAX_LENGTH = 100 DRAFT_MAX_ERRORS = 10 DRAFT_MESSAGE_MAX_LENGTH = 240 @@ -130,6 +143,22 @@ 'file_sync_source_unavailable': 'This File Sync source is not available to you.', 'workflow_conflict': 'A different workflow already uses this id.', 'workflows_unavailable': 'Personal workflows are not available for this account.', + 'merge_unavailable': 'File merging is turned off by an administrator. Remove merge from this task.', + 'merge_inputs_required': 'A merge of input documents needs at least two input documents, in merge order.', + 'merge_trigger_required': 'Merging the files a sync changed needs a File Sync trigger.', + 'merge_runner_invalid': 'A merge task merges files with code, so it has no agent runner. Remove the runner.', + 'merge_options_invalid': ( + 'These merge options cannot be used: each kind takes only its own options (' + + '; '.join( + f"{kind} takes {', '.join(MERGE_KIND_OPTION_KEYS[kind])}" + for kind in MERGE_KINDS_AVAILABLE if kind != MERGE_KIND_TABULAR + ) + + '), mapped needs columns, key_columns needs dedupe_columns, a sheet name cannot be combined ' + 'with all sheets, and a column is named once.' + ), + 'merge_format_invalid': 'Choose an output_format this merge kind creates: ' + '; '.join( + f"{kind} creates {' or '.join(MERGE_KIND_OUTPUT_FORMATS[kind])}" for kind in MERGE_KINDS_AVAILABLE + ) + '.', } DRAFT_UNAVAILABLE_REFERENCE_MESSAGE = 'A document or source in this blueprint is not available to you.' DRAFT_HANDLE_MAP_ERROR = 'The workflow draft handle map is malformed.' @@ -441,6 +470,86 @@ def _trigger_schema(): } +def _merge_schema(): + return { + 'type': 'object', + 'additionalProperties': False, + 'required': ['files'], + 'description': ( + 'Makes this task merge files with code instead of running a model. kind tabular appends the ' + 'rows of CSV and Excel files into one CSV or Excel file; workbook puts each CSV or Excel file ' + 'on its own sheet of one Excel workbook; pdf joins PDFs, in order, into one PDF; docx appends ' + 'Word documents, in order, into one Word document; pptx appends the slides of PowerPoint decks, ' + 'in order, into one deck. files: inputs ' + "merges the task's inputs (two or more document handles) in order; changed merges the files a " + "File Sync trigger added or changed; all merges every matching file in the user's personal " + 'workspace; recent merges those added in the last recent_window_minutes. Rows are appended; ' + 'rows are never matched on a key.' + ), + 'properties': { + 'kind': {'enum': list(MERGE_KINDS_AVAILABLE)}, + 'files': {'enum': list(BLUEPRINT_MERGE_FILES)}, + 'output_format': {'enum': list(dict.fromkeys( + output_format for kind in MERGE_KINDS_AVAILABLE for output_format in MERGE_KIND_OUTPUT_FORMATS[kind] + ))}, + 'file_name': _text_schema(BLUEPRINT_MERGE_FILE_NAME_MAX_LENGTH), + 'recent_window_minutes': {'type': 'integer', 'minimum': 1, 'maximum': 1440}, + 'options': {'$ref': '#/$defs/merge_options'}, + }, + } + + +def _merge_options_schema(): + name = {'type': 'string', 'minLength': 1, 'maxLength': 256} + names = {'type': 'array', 'items': name, 'minItems': 1, 'uniqueItems': True} + return { + 'type': 'object', + 'additionalProperties': False, + 'description': ( + 'The same column, sheet, duplicate and sort settings a chat merge accepts, for kind tabular. ' + 'kind workbook takes only sheets and sheet; kind pdf takes only bookmarks; kind docx takes only ' + 'formatting, page_breaks and source_headings; kind pptx takes only formatting and sections.' + ), + 'properties': { + 'bookmarks': {'type': 'boolean'}, + 'formatting': {'enum': ['keep_source', 'use_first']}, + 'page_breaks': {'type': 'boolean'}, + 'source_headings': {'type': 'boolean'}, + 'sections': {'type': 'boolean'}, + 'schema_policy': {'enum': ['by_name', 'exact_order', 'union', 'mapped']}, + 'columns': {**names, 'maxItems': 255}, + # A list of closed objects rather than a map, so every object in the blueprint stays closed. + 'column_aliases': { + 'type': 'array', 'minItems': 1, 'maxItems': 256, + 'items': { + 'type': 'object', 'additionalProperties': False, 'required': ['column', 'aliases'], + 'properties': {'column': name, 'aliases': {**names, 'maxItems': 32}}, + }, + }, + 'sheet': {'type': 'string', 'minLength': 1, 'maxLength': 31}, + 'sheets': {'enum': ['first', 'all']}, + 'header_row': {'type': 'integer', 'minimum': 1, 'maximum': 1000}, + 'include_source_column': {'type': 'boolean'}, + 'source_column_name': {'type': 'string', 'minLength': 1, 'maxLength': 128}, + 'on_incompatible': {'enum': ['fail', 'exclude']}, + 'dedupe': {'enum': ['none', 'exact_rows', 'key_columns']}, + 'dedupe_columns': {**names, 'maxItems': 16}, + 'dedupe_keep': {'enum': ['first', 'last']}, + 'sort_by': { + 'type': 'array', 'maxItems': 3, + 'items': { + 'type': 'object', 'additionalProperties': False, 'required': ['column'], + 'properties': { + 'column': name, + 'descending': {'type': 'boolean'}, + 'value_type': {'enum': ['text', 'number', 'date']}, + }, + }, + }, + }, + } + + def _build_blueprint_schema(): return { '$schema': 'https://json-schema.org/draft/2020-12/schema', @@ -471,6 +580,7 @@ def _build_blueprint_schema(): 'type': 'array', 'maxItems': BLUEPRINT_TASK_MAX_INPUTS, 'uniqueItems': True, 'items': {'$ref': '#/$defs/handle'}, }, + 'merge': {'$ref': '#/$defs/merge'}, }, }, }, @@ -502,6 +612,8 @@ def _build_blueprint_schema(): ], }, 'file_sync_schedule': _file_sync_schedule_schema(), + 'merge': _merge_schema(), + 'merge_options': _merge_options_schema(), }, } @@ -798,6 +910,99 @@ def _alert_fields(alerts, workflow_id): } +def _merge_options(options, *, aliases_as_pairs=False): + """A blueprint's merge options in the stored form: column aliases become a column-to-names map. + + Validation reads the aliases as ordered pairs, so a column named twice is reported rather + than silently overwritten. + """ + options = copy.deepcopy(options or {}) + if 'column_aliases' in options: + pairs = [(entry['column'], list(entry['aliases'])) for entry in options['column_aliases']] + options['column_aliases'] = pairs if aliases_as_pairs else dict(pairs) + return options + + +def _merge_kind(merge): + return merge.get('kind') or MERGE_KIND_TABULAR + + +def _merge_document_action(merge, task, handles): + """The workflow Merge action a blueprint task describes; its files are never reference documents.""" + kind = _merge_kind(merge) + action = { + 'type': DOCUMENT_ACTION_TYPE_MERGE, + 'merge_kind': kind, + 'output_format': merge.get('output_format') or MERGE_KIND_OUTPUT_FORMATS[kind][0], + 'document_ids': [], + 'active_group_ids': [], + 'active_public_workspace_id': [], + } + if merge.get('file_name'): + action['output_file_name'] = merge['file_name'] + if merge.get('options'): + action['merge_options'] = _merge_options(merge['options']) + files = merge['files'] + if files == 'inputs': + documents = [handles['documents'][handle] for handle in task.get('inputs') or []] + scopes = {document['scope_type'] for document in documents} + action.update({ + 'target_mode': 'selected', + 'document_ids': [document['document_id'] for document in documents], + 'doc_scope': next(iter(scopes)) if len(scopes) == 1 else 'all', + 'active_group_ids': list(dict.fromkeys( + document['scope_id'] for document in documents if document['scope_type'] == FILE_SYNC_SCOPE_GROUP + )), + 'active_public_workspace_id': list(dict.fromkeys( + document['scope_id'] for document in documents if document['scope_type'] == FILE_SYNC_SCOPE_PUBLIC + )), + }) + elif files == 'changed': + action.update({'target_mode': 'changed', 'doc_scope': 'all'}) + else: + # A chat-created workflow is personal, so run-time discovery stays in the personal workspace. + action.update({'target_mode': files, 'doc_scope': FILE_SYNC_SCOPE_PERSONAL}) + if files == 'recent' and merge.get('recent_window_minutes'): + action['recent_window_minutes'] = merge['recent_window_minutes'] + return action + + +def _merge_errors(blueprint, settings): + """Merge task rules that read no storage, so a planner can repair them in one round.""" + errors = [] + merging = [(index, task) for index, task in enumerate(blueprint['tasks']) if task.get('merge')] + if not merging: + return errors + if not is_document_action_enabled(DOCUMENT_ACTION_TYPE_MERGE, settings=settings): + return [draft_error('merge_unavailable', ('tasks', index, 'merge')) for index, _ in merging] + + for index, task in merging: + merge = task['merge'] + kind = _merge_kind(merge) + if (task.get('runner') or {}).get('type') == 'agent': + errors.append(draft_error('merge_runner_invalid', ('tasks', index, 'runner'))) + if merge['files'] == 'inputs' and len(task.get('inputs') or []) < 2: + errors.append(draft_error('merge_inputs_required', ('tasks', index, 'inputs'))) + if merge['files'] == 'changed' and blueprint['trigger']['type'] != 'file_sync': + errors.append(draft_error('merge_trigger_required', ('tasks', index, 'merge', 'files'))) + if merge.get('output_format') and merge['output_format'] not in MERGE_KIND_OUTPUT_FORMATS[kind]: + errors.append(draft_error('merge_format_invalid', ('tasks', index, 'merge', 'output_format'))) + continue + try: + # The save's own rules, including the merge engine's, for this kind of merge. + normalize_document_action_config( + { + 'type': DOCUMENT_ACTION_TYPE_MERGE, 'merge_kind': kind, 'target_mode': 'all', + 'merge_options': _merge_options(merge.get('options'), aliases_as_pairs=True), + }, + allowed_action_types={DOCUMENT_ACTION_TYPE_MERGE}, + ) + except ValueError: + # Engine messages can quote column names from the blueprint; draft errors never echo input. + errors.append(draft_error('merge_options_invalid', ('tasks', index, 'merge', 'options'))) + return errors + + def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, enabled): trigger = blueprint['trigger'] analyze_changed_documents = ( @@ -809,7 +1014,8 @@ def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, en tasks = [] for index, task in enumerate(blueprint['tasks']): task_reference_ids = [] - for handle in task.get('inputs') or []: + merge = task.get('merge') + for handle in [] if merge else task.get('inputs') or []: if handle not in reference_ids: reference_ids[handle] = _derived_id(workflow_id, 'reference', handle) references.append({'id': reference_ids[handle], 'name': handle, **handles['documents'][handle]}) @@ -820,7 +1026,9 @@ def _blueprint_payload(blueprint, handles, *, workflow_id, user_id, settings, en else: # The workflow runs on the default model; a blueprint never names one. task_runner = {'type': 'inherit'} - if index == 0 and analyze_changed_documents: + if merge: + document_action = _merge_document_action(merge, task, handles) + elif index == 0 and analyze_changed_documents: # The first task of a File Sync workflow analyzes the documents each sync changed. document_action = {'type': DOCUMENT_ACTION_TYPE_ANALYZE, 'document_ids': []} else: @@ -1028,6 +1236,7 @@ def _checked_blueprint(user_id, blueprint, handles, *, workflow_id, settings, us # Collected together, so one repair round can fix every problem at once. errors = [ *_schedule_errors(prepared['trigger'], settings), + *_merge_errors(prepared, settings), *_agent_errors(uses, handles, user_id=user_id, settings=settings, reader=reader), *_document_errors(uses, handles, user_id=user_id, resolver=resolver), *_source_errors(uses, handles, user_id=user_id, settings=settings, user_info=user_info), @@ -1059,6 +1268,7 @@ def check_workflow_blueprint(blueprint, *, settings, handle_names=None): if len(prepared['tasks']) > task_limit: return None, [draft_error('too_many_tasks', ('tasks',), f'This workflow can have up to {task_limit} tasks.')] errors = list(_schedule_errors(prepared['trigger'], settings)) + errors.extend(_merge_errors(prepared, settings)) if handle_names is not None: names = {kind: set(handle_names.get(kind) or ()) for kind in BLUEPRINT_HANDLE_KINDS} errors = [*_unknown_handle_errors(prepared, names), *errors] diff --git a/application/single_app/functions_workflow_editor.py b/application/single_app/functions_workflow_editor.py index e4e3adea2..1ee456be7 100644 --- a/application/single_app/functions_workflow_editor.py +++ b/application/single_app/functions_workflow_editor.py @@ -26,7 +26,8 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks agents, endpoints, default_model=None, max_loop_items=WORKFLOW_LOOP_ITEMS_DEFAULT, max_repeat_iterations=WORKFLOW_REPEAT_ITERATIONS_DEFAULT, - min_schedule_interval_seconds=WORKFLOW_MIN_SCHEDULE_INTERVAL_DEFAULT): + min_schedule_interval_seconds=WORKFLOW_MIN_SCHEDULE_INTERVAL_DEFAULT, + merge=None): if scope_type not in {"personal", "group"}: raise ValueError("Unsupported workflow editor scope.") agent_options = [ @@ -65,7 +66,7 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks }) default_model = default_model or {} default_model_valid = bool(default_model.get("valid")) - return { + options = { "definition_version": WORKFLOW_DEFINITION_VERSION, "supported_definition_versions": [1, 2, 3], "supported_node_kinds": ["task", "if", "route", "for_each", "collect", "repeat_until"], @@ -100,11 +101,24 @@ def build_workflow_editor_options(*, scope_type, scope_id, can_manage, max_tasks "loop_eligible": default_model_valid, }, } + if merge is not None: + # Whether a task may merge files, and how many one run may merge, as the save will check. + options["document_actions"] = {"merge": { + "enabled": bool(merge.get("enabled")), + "workflow_max_documents": int(merge["workflow_max_documents"]), + }} + return options def get_workflow_editor_options(user_id, settings, *, group_id=""): # Existing stores own agent/model eligibility and initialize app services. # Import them only at this already-authorized request boundary. + from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_WORKFLOW, + DOCUMENT_ACTION_TYPE_MERGE, + get_document_action_max_documents, + is_document_action_enabled, + ) from functions_group import assert_group_role from functions_group_workflows import ( GROUP_WORKFLOW_MEMBER_ROLES, @@ -141,4 +155,10 @@ def get_workflow_editor_options(user_id, settings, *, group_id=""): max_loop_items=get_workflow_max_loop_items(settings), max_repeat_iterations=get_workflow_max_repeat_iterations(settings), min_schedule_interval_seconds=get_workflow_min_schedule_interval_seconds(settings), + merge={ + "enabled": is_document_action_enabled(DOCUMENT_ACTION_TYPE_MERGE, settings=settings), + "workflow_max_documents": get_document_action_max_documents( + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, + ), + }, ) diff --git a/application/single_app/functions_workflow_merge.py b/application/single_app/functions_workflow_merge.py new file mode 100644 index 000000000..e9c95360f --- /dev/null +++ b/application/single_app/functions_workflow_merge.py @@ -0,0 +1,572 @@ +# functions_workflow_merge.py +"""Workflow Merge tasks: combine many authorized files into one downloadable file. + +Version: 0.261.242 +Implemented in: 0.261.241 +PDF and workbook merges added in: 0.261.242 + +A workflow task whose document action is ``merge`` runs here instead of a model or agent. +The runner says which files to merge: the files selected on the task, in that order, or +files found when the run starts (every matching file in a workspace, recently added files, +or the files a File Sync run changed), ordered by file name. This module authorizes every +file again through the source manifest and reads each original through the screening-aware +byte reader. Spreadsheet rows are appended by the tabular merge engine and rendered as one +CSV or Excel file by the shared generated-file export framework; PDFs and workbooks are +assembled by the document merge engine. The file is attached to the run's conversation. +No model is called. A resumed run repeats the merge from its first file; merging is +deterministic, so the resulting file is the same, and the artifact idempotency key keeps +one copy. +""" + +import hashlib +import os + +from functions_appinsights import log_event +from functions_document_actions import ( + DOCUMENT_ACTION_CONTEXT_WORKFLOW, + DOCUMENT_ACTION_TARGET_MODE_ALL, + DOCUMENT_ACTION_TARGET_MODE_RECENT, + DOCUMENT_ACTION_TARGET_MODE_SELECTED, + MERGE_KIND_DOCX, + MERGE_KIND_EXTENSIONS, + MERGE_KIND_LABELS, + MERGE_KIND_OUTPUT_FORMATS, + MERGE_KIND_PDF, + MERGE_KIND_PPTX, + MERGE_KIND_TABULAR, + MERGE_KIND_WORKBOOK, + MERGE_KINDS_AVAILABLE, + MERGE_TARGET_MODE_CHANGED, +) +from functions_generated_export_contracts import ( + GeneratedFileExportError, + GeneratedFileExportReadiness, + GeneratedFileExportRequest, +) +from functions_mixed_source_orchestration import AUTHORIZATION_STATUS_AUTHORIZED, SOURCE_KIND_TABULAR +from functions_orchestration_merge import ( + build_document_merge_parts, + build_tabular_merge_sources, + tabular_merge_limits, +) +from functions_tabular_merge import ( + TabularMergeCancelled, + TabularMergeError, + merge_tabular_sources, + tabular_merge_options_from_arguments, +) + + +WORKFLOW_MERGE_CAPABILITY = "file_merge" +WORKFLOW_MERGE_DEFAULT_FILE_NAME = "merged" +# A workflow run may assemble more than one chat turn; Excel and PowerPoint cap what opens well. +WORKFLOW_MERGE_MAX_PAGES = 10_000 +WORKFLOW_MERGE_MAX_SLIDES = 2_000 +WORKFLOW_MERGE_MAX_SHEETS = 500 +_EXPORT_PROFILES = { + "csv": "exact_tabular_records_v1", + "xlsx": "exact_tabular_workbook_v1", +} +_DOCUMENT_OPTION_KEYS = ("bookmarks", "sheets", "sheet", "formatting", "page_breaks", "source_headings", "sections") +_KIND_FILE_DESCRIPTIONS = { + MERGE_KIND_TABULAR: "CSV or Excel file (.csv, .xlsx, .xlsm or .xls)", + MERGE_KIND_WORKBOOK: "CSV or Excel file (.csv, .xlsx, .xlsm or .xls)", + MERGE_KIND_PDF: "PDF file (.pdf)", + MERGE_KIND_DOCX: "Word document (.docx)", + MERGE_KIND_PPTX: "PowerPoint deck (.pptx)", +} +_MAX_LISTED_FILES = 20 +_MAX_LISTED_COLUMNS = 12 +_EXPORT_FAILURES = { + "record_limit": "The merged table has more rows than one file can hold. Merge fewer files per run.", + "value_limit": "The merged table is larger than one generated file can hold. Merge fewer files per run.", +} +_OFFICE_FAILURES = { + "limit_exceeded": ( + "The merged table is too large for an Excel file (at most 1,048,575 rows, 5,000,000 cells " + "and 32 MB). Choose CSV output instead." + ), +} + + +class WorkflowMergeError(ValueError): + """A merge task that cannot run; the message is shown on the failed task.""" + + +class WorkflowMergeAccessError(PermissionError): + """A file the merge needs is not available to the workflow's owner; the message is fixed text.""" + + +class WorkflowMergeCancelled(RuntimeError): + """The run was cancelled while the merge was in progress.""" + + +class _MergedRecordsSource: + """The merged rows as a complete record source for the shared CSV and XLSX renderers.""" + + kind = "records" + + def __init__(self, merge_result): + self._result = merge_result + self.record_count = merge_result.row_count + self.columns = tuple(merge_result.columns) + self.readiness = GeneratedFileExportReadiness() + + def iter_records(self): + return self._result.iter_records() + + def recheck(self): + if self._result.row_count != self.record_count or tuple(self._result.columns) != self.columns: + raise GeneratedFileExportError("invalid_source", "The merged rows changed while rendering.") + + +def merge_output_file_name(action_config, output_format): + """The file name a merge task creates, from its configured base name.""" + base = str((action_config or {}).get("output_file_name") or "").strip() or WORKFLOW_MERGE_DEFAULT_FILE_NAME + return f"{base}.{output_format}" + + +def _target_description(target_mode): + return { + DOCUMENT_ACTION_TARGET_MODE_ALL: "No files in the workspace matched this merge", + DOCUMENT_ACTION_TARGET_MODE_RECENT: "No files were added or updated in the merge's time window", + MERGE_TARGET_MODE_CHANGED: "No new or changed files arrived with this sync", + }.get(target_mode, "No files matched this merge") + + +def _nothing_to_merge_result(target_mode): + return { + "reply": f"{_target_description(target_mode)}, so there was nothing to merge and no file was created.", + "generated_analysis_artifacts": [], + "generated_tabular_outputs": [], + "token_usage": {}, + "model_deployment_name": None, + "provider": None, + "merge_summary": {"status": "nothing_to_merge", "files": 0, "rows": 0}, + } + + +def _listed(values, limit): + values = [str(value) for value in values] + shown = ", ".join(values[:limit]) + if len(values) > limit: + shown += f" and {len(values) - limit} more" + return shown + + +def build_merge_reply(report, file_name, *, skipped_files=(), unavailable_files=()): + """A short Markdown summary of what the merge produced and what it left out.""" + totals = report.get("totals") or {} + lines = [ + f"Merged **{totals.get('rows', 0):,} row(s)** from **{totals.get('sources', 0)} file(s)** into " + f"**{file_name}** with {totals.get('columns', 0)} column(s).", + "", + ] + details = [] + if totals.get("duplicates_removed"): + details.append(f"Removed {totals['duplicates_removed']:,} duplicate row(s).") + left_out = [ + entry for entry in report.get("sources") or [] + if entry.get("status") in ("excluded", "skipped") + ] + if left_out: + labels = [ + f"{entry.get('file_name')}" + (f" ({entry.get('sheet')})" if entry.get("sheet") else "") + for entry in left_out + ] + details.append(f"Left out {len(left_out)} file(s) or sheet(s): {_listed(labels, _MAX_LISTED_FILES)}.") + if skipped_files: + details.append( + f"Skipped {len(skipped_files)} file(s) that aren't the right type: " + f"{_listed(skipped_files, _MAX_LISTED_FILES)}." + ) + if unavailable_files: + details.append( + f"Skipped {len(unavailable_files)} file(s) that aren't available right now: " + f"{_listed(unavailable_files, _MAX_LISTED_FILES)}." + ) + nullable = [name for name in report.get("nullable_columns") or [] if name != report.get("sheet_column")] + if nullable: + details.append( + f"Columns some files don't have are left blank for their rows: {_listed(nullable, _MAX_LISTED_COLUMNS)}." + ) + if details: + lines.extend(f"- {detail}" for detail in details) + lines.append("") + merged = [ + entry.get("file_name") for entry in report.get("sources") or [] if entry.get("status") == "merged" + ] + lines.append(f"Files merged, in order: {_listed(dict.fromkeys(merged), _MAX_LISTED_FILES)}.") + return "\n".join(lines).strip() + + +def execute_workflow_merge( + action_config, + settings, + *, + user_id, + conversation_id, + run_id, + task_id="", + collect_documents, + resolve_manifest, + render_file, + publish_file, + byte_reader=None, + cancel_requested=None, + report_progress=None, +): + """Merge a workflow task's files and attach the merged file to the run conversation. + + ``collect_documents(action_config, extensions, max_documents)`` returns document ids, in + merge order, for files found at run time. ``resolve_manifest`` authorizes ids again. + ``render_file(source, request, max_output_bytes)`` renders spreadsheet rows through the + shared export framework, and ``publish_file(...)`` attaches the file; both are injected so + the merge stays testable. Document kinds are assembled by the document merge engine. + """ + action_config = action_config if isinstance(action_config, dict) else {} + kind = action_config.get("merge_kind") or MERGE_KIND_TABULAR + if kind not in MERGE_KINDS_AVAILABLE: + raise WorkflowMergeError(f"Merging {MERGE_KIND_LABELS.get(kind, kind)} files is not available yet.") + target_mode = action_config.get("target_mode") or DOCUMENT_ACTION_TARGET_MODE_SELECTED + output_formats = MERGE_KIND_OUTPUT_FORMATS[kind] + output_format = action_config.get("output_format") or output_formats[0] + if output_format not in output_formats: + raise WorkflowMergeError( + f"A {MERGE_KIND_LABELS[kind]} merge creates {' or '.join(value.upper() for value in output_formats)} files." + ) + limits = tabular_merge_limits(settings, execution_context=DOCUMENT_ACTION_CONTEXT_WORKFLOW) + extensions = MERGE_KIND_EXTENSIONS[kind] + selected = target_mode == DOCUMENT_ACTION_TARGET_MODE_SELECTED + + def check_cancel(): + if callable(cancel_requested) and cancel_requested(): + raise WorkflowMergeCancelled("The merge was cancelled.") + + check_cancel() + if target_mode in (DOCUMENT_ACTION_TARGET_MODE_ALL, DOCUMENT_ACTION_TARGET_MODE_RECENT): + document_ids = list(collect_documents(action_config, extensions, limits.max_sources)) + else: + document_ids = list(action_config.get("document_ids") or []) + if not document_ids: + if selected: + raise WorkflowMergeError("Select at least two files to merge.") + return _nothing_to_merge_result(target_mode) + if len(document_ids) > limits.max_sources: + raise WorkflowMergeError( + f"{len(document_ids)} files match this merge; at most {limits.max_sources} can be merged in one " + "workflow run. Narrow the files or ask an administrator to raise the Merge workflow file limit." + ) + + check_cancel() + manifest = resolve_manifest( + document_ids, user_id=user_id, conversation_id=conversation_id, + doc_scope=action_config.get("doc_scope", "all"), + active_group_ids=action_config.get("active_group_ids"), + active_public_workspace_ids=action_config.get("active_public_workspace_id"), + cancel_requested=cancel_requested, request_correlation_id=run_id, + ) + if ( + not isinstance(manifest, list) + or [source.get("document_id") for source in manifest if isinstance(source, dict)] != document_ids + ): + raise WorkflowMergeAccessError("The files in this merge could not be authorized.") + unavailable = [ + str(source.get("file_name") or source.get("document_id")) for source in manifest + if source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED + ] + if unavailable and selected: + raise WorkflowMergeAccessError( + "A file selected for this merge is no longer available to the workflow's owner." + ) + + usable = [] + skipped = [] + for source in manifest: + if source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED: + continue + extension = os.path.splitext(str(source.get("file_name") or ""))[1].strip().lower() + if extension in extensions and (kind != MERGE_KIND_TABULAR or source.get("source_kind") == SOURCE_KIND_TABULAR): + usable.append(source) + else: + skipped.append(str(source.get("file_name") or source.get("document_id"))) + if skipped and selected: + raise WorkflowMergeError( + f"{_listed(skipped, 5)} isn't a {_KIND_FILE_DESCRIPTIONS[kind]}, so it can't be merged." + ) + if not usable: + return _nothing_to_merge_result(target_mode) + + def progress(update): + if callable(report_progress): + report_progress(update) + + run = { + "kind": kind, "usable": usable, "skipped": skipped, "unavailable": unavailable, + "user_id": user_id, "conversation_id": conversation_id, "run_id": run_id, "task_id": task_id, + "file_name": merge_output_file_name(action_config, output_format), "output_format": output_format, + "min_sources": 2 if selected else 1, "max_sources": limits.max_sources, + } + if kind == MERGE_KIND_TABULAR: + result = _merge_rows( + run, action_config, settings, limits, render_file=render_file, publish_file=publish_file, + byte_reader=byte_reader, cancel_requested=cancel_requested, check_cancel=check_cancel, + progress=progress, + ) + else: + result = _merge_files( + run, action_config, settings, publish_file=publish_file, byte_reader=byte_reader, + cancel_requested=cancel_requested, check_cancel=check_cancel, progress=progress, + ) + log_event( + "[WORKFLOW_MERGE] Merged workflow files.", + extra={ + "user_id": user_id, "run_id": run_id, "task_id": task_id, "kind": kind, + "files": len(usable), "output_format": output_format, "skipped": len(skipped), + "unavailable": len(unavailable), + }, + ) + return result + + +def _idempotency_key(run, content_sha256): + # A replay of the same merge reuses its file; one over files that changed since gets its own. + return f"workflow-merge:{run['run_id']}:{run['task_id'] or 'task'}:{content_sha256}" + + +def _merge_rows(run, action_config, settings, limits, *, render_file, publish_file, byte_reader, cancel_requested, + check_cancel, progress): + """Append the rows of spreadsheet files into one CSV or Excel file.""" + sources = build_tabular_merge_sources(run["usable"], run["user_id"], byte_reader=byte_reader) + output_format = run["output_format"] + try: + options = tabular_merge_options_from_arguments(action_config.get("merge_options") or {}) + with merge_tabular_sources( + sources, options=options, limits=limits, min_sources=run["min_sources"], + cancel_requested=lambda: bool(callable(cancel_requested) and cancel_requested()), + on_progress=progress, + ) as merged: + check_cancel() + report = merged.report + request = GeneratedFileExportRequest(output_format, _EXPORT_PROFILES[output_format]) + with render_file(_MergedRecordsSource(merged), request, _max_artifact_bytes(settings)) as rendered: + check_cancel() + artifact = publish_file( + user_id=run["user_id"], conversation_id=run["conversation_id"], file_name=run["file_name"], + output_format=output_format, rendered=rendered, row_count=merged.row_count, + summary=f"Merged {merged.row_count:,} row(s) from {report['totals']['sources']} file(s).", + idempotency_key=_idempotency_key(run, str(getattr(rendered, "content_sha256", "") or "")), + ) + except TabularMergeCancelled as exc: + raise WorkflowMergeCancelled("The merge was cancelled.") from exc + except TabularMergeError as exc: + # The owner's own file and column names make the failure actionable on the task. + raise WorkflowMergeError(exc.message) from exc + except GeneratedFileExportError as exc: + raise WorkflowMergeError( + _EXPORT_FAILURES.get(exc.code, "The merged file couldn't be created.") + ) from exc + except ValueError as exc: + # Office renderer failures carry a stable code; the module loads only when rendering. + if type(exc).__name__ != "OfficeRenderError": + raise + raise WorkflowMergeError( + _OFFICE_FAILURES.get(getattr(exc, "code", None), "The merged Excel file couldn't be created.") + ) from exc + + file_name = artifact.get("file_name") or run["file_name"] + return { + "reply": build_merge_reply( + report, file_name, skipped_files=run["skipped"], unavailable_files=run["unavailable"], + ), + "generated_analysis_artifacts": [artifact], + "generated_tabular_outputs": [artifact] if output_format == "csv" else [], + "token_usage": {}, + "model_deployment_name": None, + "provider": None, + "merge_summary": { + "status": "merged", + "kind": MERGE_KIND_TABULAR, + "files": report["totals"]["sources"], + "rows": report["totals"]["rows"], + "columns": report["totals"]["columns"], + "duplicates_removed": report["totals"].get("duplicates_removed", 0), + "excluded": report["totals"].get("excluded", 0), + "skipped": len(run["skipped"]) + len(run["unavailable"]), + "file_name": file_name, + }, + } + + +class _MergedFile: + """An assembled file as the artifact upload reads it: a seekable stream, its size and digest.""" + + def __init__(self, result): + digest = hashlib.sha256() + for chunk in result.iter_chunks(): + digest.update(chunk) + self.file_content = result.open_stream() + self.size_bytes = result.size_bytes + self.content_sha256 = digest.hexdigest() + + +def _document_limits(settings, max_sources, kind): + """Workflow bounds for assembled files: more files, pages and sheets than one chat turn allows. + + A workbook is copied one file at a time to disk, so it may read up to twice the + generated-file limit. Other kinds hold their merged content in memory until the file + is written, so they keep the engine's fixed input budget whatever that limit is. + """ + from functions_document_merge import DocumentMergeLimits + + artifact_bytes = _max_artifact_bytes(settings) + total_input_bytes = DocumentMergeLimits.max_total_input_bytes + if kind == MERGE_KIND_WORKBOOK: + total_input_bytes = max(total_input_bytes, 2 * artifact_bytes) + return DocumentMergeLimits( + max_parts=max(2, max_sources), + max_output_bytes=artifact_bytes, + max_total_input_bytes=total_input_bytes, + max_pages=WORKFLOW_MERGE_MAX_PAGES, + max_slides=WORKFLOW_MERGE_MAX_SLIDES, + max_sheets=WORKFLOW_MERGE_MAX_SHEETS, + ) + + +def _merge_files(run, action_config, settings, *, publish_file, byte_reader, cancel_requested, check_cancel, + progress): + """Assemble PDF, Word, PowerPoint or workbook files, in order, into one file.""" + from functions_document_merge import ( + DocumentMergeCancelled, + DocumentMergeError, + DocumentMergeOptions, + merge_documents, + ) + + kind = run["kind"] + merge_options = action_config.get("merge_options") or {} + parts = build_document_merge_parts(run["usable"], run["user_id"], byte_reader=byte_reader) + try: + options = DocumentMergeOptions(**{ + key: merge_options[key] for key in _DOCUMENT_OPTION_KEYS if key in merge_options + }) + result = merge_documents( + kind, parts, options=options, limits=_document_limits(settings, run["max_sources"], kind), + cancel_requested=lambda: bool(callable(cancel_requested) and cancel_requested()), + on_progress=progress, min_parts=run["min_sources"], + ) + except DocumentMergeCancelled as exc: + raise WorkflowMergeCancelled("The merge was cancelled.") from exc + except DocumentMergeError as exc: + # The owner's own file names make the failure actionable on the task. + raise WorkflowMergeError(exc.message) from exc + + with result: + check_cancel() + report = result.report + merged_file = _MergedFile(result) + merged = [entry for entry in report.get("parts") or [] if entry.get("status") == "merged"] + artifact = publish_file( + user_id=run["user_id"], conversation_id=run["conversation_id"], file_name=run["file_name"], + output_format=run["output_format"], rendered=merged_file, row_count=None, + summary=f"Merged {len(merged)} file(s) into one {MERGE_KIND_LABELS[kind]} file.", + idempotency_key=_idempotency_key(run, merged_file.content_sha256), + ) + + file_name = artifact.get("file_name") or run["file_name"] + totals = report.get("totals") or {} + return { + "reply": build_document_merge_reply( + report, file_name, skipped_files=run["skipped"], unavailable_files=run["unavailable"], + ), + "generated_analysis_artifacts": [artifact], + "generated_tabular_outputs": [], + "token_usage": {}, + "model_deployment_name": None, + "provider": None, + "merge_summary": { + "status": "merged", + "kind": kind, + "files": len(merged), + "pages": totals.get("pages", 0), + "slides": totals.get("slides", 0), + "sheets": totals.get("sheets", 0), + "skipped": len(run["skipped"]) + len(run["unavailable"]), + "file_name": file_name, + }, + } + + +def build_document_merge_reply(report, file_name, *, skipped_files=(), unavailable_files=()): + """A short Markdown summary of an assembled file and what it left out or changed.""" + totals = report.get("totals") or {} + parts = report.get("parts") or [] + merged = [entry.get("file_name") for entry in parts if entry.get("status") == "merged"] + size = { + "pdf": ("pages", "page(s)"), "pptx": ("slides", "slide(s)"), "workbook": ("sheets", "sheet(s)"), + }.get(report.get("kind")) + measure = f" with {totals.get(size[0], 0):,} {size[1]}" if size else "" + lines = [f"Merged **{len(merged)} file(s)** into **{file_name}**{measure}.", ""] + details = [] + for warning in report.get("warnings") or []: + details.append(str(warning.get("message") or "")) + for entry in parts: + for warning in entry.get("warnings") or []: + details.append(f"{entry.get('file_name')}: {warning.get('message')}") + if skipped_files: + details.append( + f"Skipped {len(skipped_files)} file(s) that aren't the right type: " + f"{_listed(skipped_files, _MAX_LISTED_FILES)}." + ) + if unavailable_files: + details.append( + f"Skipped {len(unavailable_files)} file(s) that aren't available right now: " + f"{_listed(unavailable_files, _MAX_LISTED_FILES)}." + ) + details.extend(str(limitation) for limitation in report.get("limitations") or []) + details = [detail for detail in dict.fromkeys(details) if detail] + if details: + lines.extend(f"- {detail}" for detail in details) + lines.append("") + lines.append(f"Files merged, in order: {_listed(merged, _MAX_LISTED_FILES)}.") + return "\n".join(lines).strip() + + +def _max_artifact_bytes(settings): + try: + megabytes = max(1, int((settings or {}).get("max_generated_chat_artifact_size_mb", 500))) + except (TypeError, ValueError): + megabytes = 500 + return megabytes * 1024 * 1024 + + +def collect_merge_candidates(documents, extensions, max_documents): + """Current, matching documents ordered by file name; more than the limit fails, never truncates.""" + matches = {} + for document in documents or (): + document_id = str(document.get("id") or document.get("document_id") or "").strip() + file_name = str(document.get("file_name") or "").strip() + if not document_id or document_id in matches: + continue + if os.path.splitext(file_name)[1].lower() not in extensions: + continue + matches[document_id] = (file_name.casefold(), file_name, document_id) + if len(matches) > max_documents: + raise WorkflowMergeError( + f"More than {max_documents} files match this merge; at most {max_documents} can be merged in one " + "workflow run. Narrow the files or ask an administrator to raise the Merge workflow file limit." + ) + return [document_id for _, _, document_id in sorted(matches.values())] + + +__all__ = [ + "WORKFLOW_MERGE_CAPABILITY", + "WorkflowMergeAccessError", + "WorkflowMergeCancelled", + "WorkflowMergeError", + "build_document_merge_reply", + "build_merge_reply", + "collect_merge_candidates", + "execute_workflow_merge", + "merge_output_file_name", +] diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 5d9ce7060..6b93ddf57 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -119,9 +119,11 @@ DOCUMENT_ACTION_TARGET_MODE_RECENT, DOCUMENT_ACTION_TYPE_COMPARISON, DOCUMENT_ACTION_TYPE_ANALYZE, + DOCUMENT_ACTION_TYPE_MERGE, DOCUMENT_ACTION_TYPE_NONE, DOCUMENT_ACTION_TYPE_SEARCH, DEFAULT_RECENT_DOCUMENT_WINDOW_MINUTES, + MERGE_TARGET_MODE_CHANGED, build_analyze_config, get_document_action_config, get_document_action_max_documents, @@ -288,6 +290,7 @@ from functions_simplechat_operations import ( delete_generated_chat_artifact_for_current_user, delete_generated_chat_artifact_for_user, + upload_generated_file_artifact_stream_for_user, upload_generated_analysis_artifact_for_current_user, upload_generated_analysis_artifact_for_user, ) @@ -6490,7 +6493,8 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru raw_agent_citations = list(result.get('agent_citations') or []) xsd_generation_contract = result.get('_xsd_generation_contract') generated_file_output = None - if not xsd_generation_contract: + # A merge task's merged file is its output; its instructions never ask for a second file. + if not xsd_generation_contract and not result.get('merge_summary'): generated_file_output = _maybe_create_workflow_generated_file_output( workflow=workflow, conversation_id=conversation.get('id'), @@ -6840,7 +6844,7 @@ def _get_document_action_config(workflow): DOCUMENT_ACTION_CONTEXT_WORKFLOW, settings=settings, ), - allowed_action_types=get_enabled_document_action_types(settings=settings), + allowed_action_types=get_enabled_document_action_types(settings=settings, include_merge=True), ) @@ -7741,8 +7745,21 @@ def _format_workflow_file_sync_context(file_sync_result): def _apply_file_sync_changed_documents_to_action(action_config, changed_document_ids, group_ids, public_workspace_ids): - """Point an analyze document action at the documents File Sync just changed.""" + """Point an analyze document action, or a merge of changed files, at the documents File Sync just changed.""" action_config = action_config if isinstance(action_config, dict) else {} + if action_config.get('type') == DOCUMENT_ACTION_TYPE_MERGE: + if action_config.get('target_mode') != MERGE_TARGET_MODE_CHANGED: + return None + # A merge with no changed files reports that there was nothing to merge. + updated_merge = dict(action_config) + updated_merge.update({ + 'document_ids': list(changed_document_ids), + 'doc_scope': 'all', + 'active_group_ids': list(group_ids), + 'active_public_workspace_id': list(public_workspace_ids), + 'merge_targets_resolved': True, + }) + return updated_merge if action_config.get('type') != DOCUMENT_ACTION_TYPE_ANALYZE: return None @@ -9859,6 +9876,195 @@ def record_native_token_usage(token_usage): } +def _query_merge_candidate_documents(container, scope_clause, scope_parameters, extensions, cutoff_ts=None): + parameters = list(scope_parameters) + extension_clauses = [] + for index, extension in enumerate(extensions): + parameter = f'@merge_extension_{index}' + extension_clauses.append(f'ENDSWITH(LOWER(c.file_name), {parameter})') + parameters.append({'name': parameter, 'value': extension}) + query = f"SELECT * FROM c WHERE {scope_clause} AND ({' OR '.join(extension_clauses)})" + if cutoff_ts is not None: + query += ' AND c._ts >= @cutoff_ts' + parameters.append({'name': '@cutoff_ts', 'value': cutoff_ts}) + return list(container.query_items(query=query, parameters=parameters, enable_cross_partition_query=True)) + + +def _collect_merge_workflow_documents(workflow, action_config, settings, extensions): + """Current documents of a merge's file types in the task's authorized workspace scopes. + + Every match is returned; the merge orders them and refuses more than its file limit + rather than silently merging only some of them. + """ + user_id = str(workflow.get('user_id') or '').strip() + workflow_group_id = _get_workflow_group_id(workflow) + doc_scope = normalize_search_scope(action_config.get('doc_scope')) + active_group_ids = normalize_search_id_list(action_config.get('active_group_ids')) + if workflow_group_id: + assert_group_role( + user_id, + workflow_group_id, + allowed_roles=("Owner", "Admin", "DocumentManager", "User"), + ) + active_group_ids = [workflow_group_id] + doc_scope = 'group' + else: + active_group_ids = _resolve_recent_authorized_group_ids(user_id, active_group_ids) + active_public_workspace_ids = normalize_search_id_list(action_config.get('active_public_workspace_id')) + if not workflow_group_id: + active_public_workspace_ids = _resolve_recent_authorized_public_workspace_ids( + user_id, active_public_workspace_ids, + ) + cutoff_ts = None + if action_config.get('target_mode') == DOCUMENT_ACTION_TARGET_MODE_RECENT: + window_minutes = _coerce_workflow_recent_window_minutes(action_config.get('recent_window_minutes')) + cutoff_ts = int(datetime.now(timezone.utc).timestamp()) - (window_minutes * 60) + + documents = [] + if doc_scope in {'personal', 'all'} and not workflow_group_id and user_id: + documents.extend(_query_merge_candidate_documents( + cosmos_user_documents_container, 'c.user_id = @user_id', + [{'name': '@user_id', 'value': user_id}], extensions, cutoff_ts, + )) + if doc_scope in {'group', 'all'}: + for group_id in active_group_ids: + documents.extend(_query_merge_candidate_documents( + cosmos_group_documents_container, 'c.group_id = @group_id', + [{'name': '@group_id', 'value': group_id}], extensions, cutoff_ts, + )) + if doc_scope in {'public', 'all'} and not workflow_group_id: + for workspace_id in active_public_workspace_ids: + documents.extend(_query_merge_candidate_documents( + cosmos_public_documents_container, 'c.public_workspace_id = @workspace_id', + [{'name': '@workspace_id', 'value': workspace_id}], extensions, cutoff_ts, + )) + return select_current_documents(documents) + + +def _throttled_cancellation_check(check, interval_seconds=2.0): + """Ask whether a run was cancelled at most once per interval; a cancellation sticks.""" + state = {'checked_at': None, 'cancelled': False} + + def cancelled(): + now = time.monotonic() + if not state['cancelled'] and ( + state['checked_at'] is None or now - state['checked_at'] >= interval_seconds + ): + state['checked_at'] = now + state['cancelled'] = bool(check()) + return state['cancelled'] + + return cancelled + + +def _execute_document_merge_workflow( + workflow, + action_config, + settings, + conversation_id='', + run_id=None, + thought_tracker=None, +): + """Run a task's Merge document action: many files into one file, without a model.""" + # The merge engine loads only when a workflow merges files. + from functions_workflow_merge import ( + WORKFLOW_MERGE_CAPABILITY, + WorkflowMergeAccessError, + WorkflowMergeCancelled, + WorkflowMergeError, + collect_merge_candidates, + execute_workflow_merge, + ) + + user_id = str(workflow.get('user_id') or '').strip() + task = workflow.get('active_task') if isinstance(workflow.get('active_task'), dict) else {} + activity_key = f"file-merge:{run_id}:{task.get('id') or 'task'}" + + def collect(action, extensions, max_documents): + documents = _collect_merge_workflow_documents(workflow, action, settings, extensions) + return collect_merge_candidates(documents, extensions, max_documents) + + def render(source, request, max_output_bytes): + return build_generated_file_export(source=source, export_request=request, max_output_bytes=max_output_bytes) + + def publish(*, user_id, conversation_id, file_name, output_format, rendered, row_count, summary, idempotency_key): + upload_result = upload_generated_file_artifact_stream_for_user( + user_id, + conversation_id, + file_name, + rendered.file_content, + rendered.size_bytes, + capability=WORKFLOW_MERGE_CAPABILITY, + output_format=output_format, + summary=summary, + artifact_idempotency_key=idempotency_key, + ) + metadata = build_generated_file_artifact_metadata( + { + 'file_name': file_name, 'output_format': output_format, 'capability': WORKFLOW_MERGE_CAPABILITY, + 'summary': summary, 'row_count': row_count, + }, + upload_result, + conversation_id, + ) + if not metadata: + raise RuntimeError('The merged file could not be attached to the workflow run.') + return metadata + + def progress(update): + _add_workflow_activity_thought( + thought_tracker, + workflow, + run_id, + step_type='file_merge', + content=f"Merged file {update.get('index')} of {update.get('total')}", + detail=', '.join( + f'{unit}={update[unit]}' for unit in ('rows', 'pages', 'slides', 'sheets') if update.get(unit) + ) or None, + activity_key=activity_key, + kind='file_merge', + title='Merge files', + status='running', + ) + + cancel_requested = _throttled_cancellation_check( + lambda: _is_workflow_run_cancellation_requested(workflow, run_id), + ) + try: + result = execute_workflow_merge( + action_config, + settings, + user_id=user_id, + conversation_id=conversation_id, + run_id=run_id, + task_id=str(task.get('id') or ''), + collect_documents=collect, + resolve_manifest=resolve_analysis_source_manifest, + render_file=render, + publish_file=publish, + cancel_requested=cancel_requested, + report_progress=progress, + ) + except WorkflowMergeCancelled as exc: + raise WorkflowRunCancelledError(WORKFLOW_RUN_CANCELLED_MESSAGE) from exc + except (WorkflowMergeError, WorkflowMergeAccessError) as exc: + # The task shows why, in terms of the owner's own files; the same merge would fail again. + raise WorkflowInputError(str(exc)) from exc + _add_workflow_activity_thought( + thought_tracker, + workflow, + run_id, + step_type='file_merge', + content='Merged files', + detail=f"files={(result.get('merge_summary') or {}).get('files', 0)}", + activity_key=activity_key, + kind='file_merge', + title='Merge files', + status='completed', + ) + return result + + def _execute_document_action_workflow( workflow, settings, @@ -10507,7 +10713,20 @@ def _execute_workflow_dispatch( execution_workflow = workflow_search_context.get('workflow') or execution_workflow document_action = _get_document_action_config(execution_workflow) - if document_action.get('type') in {DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON}: + if document_action.get('type') == DOCUMENT_ACTION_TYPE_MERGE: + execution_result = _execute_cancelable_workflow_step( + execution_workflow, + run_id, + lambda: _execute_document_merge_workflow( + execution_workflow, + document_action, + settings, + conversation_id=conversation_id, + run_id=run_id, + thought_tracker=thought_tracker, + ), + ) + elif document_action.get('type') in {DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON}: _raise_if_workflow_run_cancelled(execution_workflow, run_id) document_action = _resolve_recent_document_action_targets(execution_workflow, document_action, settings) execution_workflow = _apply_runtime_document_action_config(execution_workflow, document_action) @@ -11014,11 +11233,13 @@ def raise_if_cancelled(): 'runner': runner_audit, **({'iteration_inputs': durable.iteration_inputs} if flow_runner and durable.iteration_path else {}), } + # A merge only reads files and attaches its file under an idempotency key, so a + # durable run may repeat it instead of pausing for a review of external actions. replay_safe = ( attempt_workflow.get('runner_type') == 'model' and not attempt_workflow.get('chat_capabilities_enabled') and (attempt_workflow.get('document_action') or {}).get('type') in {None, 'none'} - ) + ) or (attempt_workflow.get('document_action') or {}).get('type') == 'merge' def dispatch_task(): nonlocal analysis_checkpoints if structured_definition: diff --git a/application/single_app/requirements.txt b/application/single_app/requirements.txt index 57dd49bc1..6510c0b16 100644 --- a/application/single_app/requirements.txt +++ b/application/single_app/requirements.txt @@ -41,6 +41,7 @@ azure-storage-queue==12.12.0 azure-keyvault-secrets==4.10.0 pypdf==6.16.1 python-docx==1.1.2 +docxcompose==2.2.0 python-pptx==1.0.2 flask-executor==1.0.0 PyMuPDF==1.25.3 diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 0d1def4aa..14c2c4861 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1705,6 +1705,33 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul 10, ), }, + 'merge': { + 'enabled': form_data.get('document_action_merge_enabled') == 'on', + 'chat_max_documents': parse_admin_int( + form_data.get('document_action_merge_chat_max_documents'), + current_document_action_capabilities.get('merge', {}).get('chat_max_documents', 10), + 'document_action_merge_chat_max_documents', + 10, + ), + 'workflow_max_documents': parse_admin_int( + form_data.get('document_action_merge_workflow_max_documents'), + current_document_action_capabilities.get('merge', {}).get('workflow_max_documents', 100), + 'document_action_merge_workflow_max_documents', + 100, + ), + 'chat_max_rows': parse_admin_int( + form_data.get('document_action_merge_chat_max_rows'), + current_document_action_capabilities.get('merge', {}).get('chat_max_rows', 250000), + 'document_action_merge_chat_max_rows', + 250000, + ), + 'workflow_max_rows': parse_admin_int( + form_data.get('document_action_merge_workflow_max_rows'), + current_document_action_capabilities.get('merge', {}).get('workflow_max_rows', 1000000), + 'document_action_merge_workflow_max_rows', + 1000000, + ), + }, } }) diff --git a/application/single_app/static/js/workspace/workspace_workflows.js b/application/single_app/static/js/workspace/workspace_workflows.js index 8548f0eb4..29be9fd03 100644 --- a/application/single_app/static/js/workspace/workspace_workflows.js +++ b/application/single_app/static/js/workspace/workspace_workflows.js @@ -238,6 +238,8 @@ const DOCUMENT_ACTION_NONE = "none"; const DOCUMENT_ACTION_SEARCH = "search"; const DOCUMENT_ACTION_ANALYZE = "analyze"; const DOCUMENT_ACTION_COMPARISON = "comparison"; +// Merge files is edited only in V2; the classic page lists it and refuses to edit it. +const DOCUMENT_ACTION_MERGE = "merge"; const DOCUMENT_ANALYSIS_MODE_COMBINED = "combined"; const DOCUMENT_ANALYSIS_MODE_PER_DOCUMENT = "per_document"; const DOCUMENT_ANALYSIS_TARGET_SELECTED = "selected"; @@ -3340,6 +3342,10 @@ function getWorkflowDocumentActionSummary(workflow) { return `Compare one source to ${rightCount || 0} ${rightCount === 1 ? "target" : "targets"}`; } + if (config.type === DOCUMENT_ACTION_MERGE) { + return "Merge files"; + } + return "No document action"; } @@ -4258,7 +4264,8 @@ function resetWorkflowForm() { } // Calendar schedules come first: the classic form edits only fixed intervals, and saving one here -// would replace the calendar schedule. +// would replace the calendar schedule. The classic form has no Merge files action either, so it +// would save a merge task as a task with no document action. function workflowNativeEditorReason(workflow) { if (!workflow || typeof workflow !== "object") { return ""; @@ -4269,6 +4276,12 @@ function workflowNativeEditorReason(workflow) { if ((workflow.definition_version !== undefined && workflow.definition_version !== 1) || workflow.flow !== undefined) { return "advanced data flow"; } + const tasks = Array.isArray(workflow.tasks) ? workflow.tasks : []; + if ([workflow.document_action, ...tasks.map((task) => task?.document_action)].some( + (action) => normalizeText(action?.type).toLowerCase() === "merge" + )) { + return "a Merge files task"; + } return ""; } diff --git a/application/single_app/templates/admin/_panes/actions.html b/application/single_app/templates/admin/_panes/actions.html index 8030031af..e447fa0cf 100644 --- a/application/single_app/templates/admin/_panes/actions.html +++ b/application/single_app/templates/admin/_panes/actions.html @@ -4,6 +4,7 @@ set in one pane is not visible to the next one. #} {% set analyze_capability = settings.document_action_capabilities.analyze %} {% set comparison_capability = settings.document_action_capabilities.comparison %} + {% set merge_capability = settings.document_action_capabilities.merge or {} %}
Microsoft 365 retrieval

Auto uses Copilot Retrieval only for verified licensed users in supported clouds. Other users use delegated Microsoft Graph. Pay-as-you-go is not used.

@@ -25,7 +26,7 @@
Document Action CapabilitiesChat: 2-300 | Workflow: 2-1000
@@ -118,6 +119,89 @@
Document Action Capabilities
+
+
+
+ + + +
+
+
+
+ + {{ merge_capability.chat_max_documents|default(10) }} +
+
+
+ +
+
+ +
+
+
+ 2 + 300 +
+
+
+
+ + {{ merge_capability.workflow_max_documents|default(100) }} +
+
+
+ +
+
+ +
+
+
+ 2 + 1000 +
+
+
+
+ + {{ merge_capability.chat_max_rows|default(250000) }} +
+
+
+ +
+
+ +
+
+
+ 1,000 + 1,000,000 +
+
+
+
+ + {{ merge_capability.workflow_max_rows|default(1000000) }} +
+
+
+ +
+
+ +
+
+
+ 1,000 + 1,000,000 +
+
+
+
+
diff --git a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx index 7c8e88f0f..fb9c75669 100644 --- a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx +++ b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx @@ -64,6 +64,7 @@ import type { import type { ImageReferenceRequest } from '../../lib/imageReferences'; import { useDocumentTitles } from '../../lib/documentTitles'; import { plannedFileSpecification, type OrchestrationExportFormat } from '../../lib/orchestrationExports'; +import { tabularArgumentEntries } from '../../lib/orchestrationMerge'; import { WORKFLOW_RESULTS_ARGUMENT_KEYS, WORKFLOW_RESULTS_CAPABILITY, @@ -104,6 +105,11 @@ const STATUS_LABELS: Record = { /** A step argument key/value the run will use, minus the document fields shown as chips. */ function readableArguments(step: OrchestrationStep): Array<[string, string]> { + // Spreadsheet merge and inspection settings are put in words, without restating defaults. + const tabular = tabularArgumentEntries(step.capability_id, step.arguments); + if (tabular) { + return tabular; + } const hidden = new Set([ 'document_ids', 'right_document_ids', @@ -429,7 +435,7 @@ export function OrchestrationRunView({ ) : null} {args.length > 0 ? ( -
+
{args.map(([key, value]) => (
diff --git a/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx b/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx index 84fa58f73..588637fc3 100644 --- a/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx +++ b/application/v2_ui/src/components/chat/WorkflowProposalCard.tsx @@ -33,6 +33,7 @@ import { type WorkflowProposalMode, type WorkflowProposalState, type WorkflowProposalSummary, + workflowProposalMergeText as mergeTaskText, } from '../../lib/workflowProposals'; const PERSONAL_SCOPE: WorkflowScope = { type: 'personal' }; @@ -250,18 +251,22 @@ function ProposalDetails({ proposal }: { proposal: WorkflowProposal }) {

  • {`${index + 1}. ${task.title || 'Untitled task'}`}

    - {task.runner === 'agent' - ? `Runs with the agent ${task.agent_name || 'chosen for it'}.` - : 'Runs with the default model.'} + {task.merge + ? mergeTaskText(task) + : task.runner === 'agent' + ? `Runs with the agent ${task.agent_name || 'chosen for it'}.` + : 'Runs with the default model.'}

    {task.requested_actions.length ? (

    {`Needs: ${actionList(task.requested_actions)}.`}

    ) : null} - {task.action_kinds.length ? ( + {task.action_kinds.length && !task.merge ? (

    {`The agent can use: ${actionList(task.action_kinds)}.`}

    ) : null} {task.inputs.length ? ( -

    {`Reads: ${task.inputs.join(', ')}.`}

    +

    + {`${task.merge ? 'Files to merge, in order' : 'Reads'}: ${task.inputs.join(', ')}.`} +

    ) : null}
    diff --git a/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx b/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx index ca9fe8584..3bdbe6f75 100644 --- a/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowFileSyncFields.tsx @@ -269,7 +269,7 @@ export function WorkflowFileSyncFields({ checked={config.use_changed_documents} disabled={disabled || !active} onChange={(checked) => update({ use_changed_documents: checked })} - description="Analyze tasks without selected documents run on the files this sync changed." + description="Analyze tasks without selected documents, and Merge files tasks set to merge the files File Sync changed, run on the files this sync changed." /> ); diff --git a/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx b/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx index 819ecaec7..ceb9b0a6e 100644 --- a/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx +++ b/application/v2_ui/src/components/workflows/WorkflowTaskFields.tsx @@ -21,14 +21,24 @@ import { } from '../../lib/workflowFlow'; import { comparisonActionFromSelection, + cleanupWorkflowMergeOptions, documentActionFromSelection, + formatWorkflowMergeColumnAliases, findWorkflowAgent, isWorkflowPublicationCompletionPolicy, isWorkflowPublicationSourceKind, + parseWorkflowMergeColumnAliases, safeWorkflowAlias, savedOutputPublicationFormats, + workflowMergeActionFromSelection, + workflowMergeDocumentLimit, + workflowMergeEnabled, + workflowMergeValidationErrors, workflowFileSyncProvidesAnalyzeTargets, workflowInputProcessingErrors, + WORKFLOW_MERGE_KIND_INPUTS, + WORKFLOW_MERGE_KIND_LABELS, + WORKFLOW_MERGE_KINDS_AVAILABLE, WORKFLOW_APPROVAL_MESSAGE_LIMIT, workflowSchemaErrors, workflowAgentKey, @@ -43,6 +53,12 @@ import { type WorkflowAgentReference, type WorkflowInputBinding, type WorkflowInputOutput, + type WorkflowMergeActionOptions, + type WorkflowMergeKind, + type WorkflowMergeOptions, + type WorkflowMergeOutputFormat, + type WorkflowMergeSortRule, + type WorkflowMergeTargetMode, type WorkflowOutputContract, type WorkflowOutputKind, type WorkflowScope, @@ -746,6 +762,9 @@ function actionMode(action: WorkflowDocumentAction | undefined): string { if (action.type === 'comparison') { return 'comparison'; } + if (action.type === 'merge') { + return 'merge'; + } if (action.type === 'search') { return Array.isArray(action.document_ids) && action.document_ids.length ? 'search_selected' @@ -766,16 +785,463 @@ function splitComparisonEvidence( return { left, right }; } +function mergeKind(action: WorkflowDocumentAction | undefined): WorkflowMergeKind { + return ['tabular', 'workbook', 'pdf', 'docx', 'pptx'].includes(String(action?.merge_kind)) + ? action?.merge_kind as WorkflowMergeKind : 'tabular'; +} + +function mergeTargetMode(action: WorkflowDocumentAction | undefined): WorkflowMergeTargetMode { + return ['selected', 'all', 'recent', 'changed'].includes(String(action?.target_mode)) + ? action?.target_mode as WorkflowMergeTargetMode : 'selected'; +} + +function mergeOutputFormat(action: WorkflowDocumentAction | undefined, kind: WorkflowMergeKind): WorkflowMergeOutputFormat { + if (kind === 'tabular' && (action?.output_format === 'xlsx' || action?.output_format === 'csv')) { + return action.output_format; + } + if (kind === 'tabular') { + return 'csv'; + } + return kind === 'workbook' ? 'xlsx' : kind; +} + +function mergeOptions(action: WorkflowDocumentAction | undefined): WorkflowMergeOptions { + return isRecord(action?.merge_options) ? { ...action.merge_options as WorkflowMergeOptions } : {}; +} + +function optionTextList(value: unknown): string { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string').join('\n') : ''; +} + +function setOptionTextList(options: WorkflowMergeOptions, key: keyof WorkflowMergeOptions, value: string): WorkflowMergeOptions { + const next = { ...options }; + const values = value.split(/\r?\n|,/).map((item) => item.trim()).filter(Boolean); + if (values.length) { + next[key] = values as never; + } else { + delete next[key]; + } + return next; +} + +function sortRules(value: unknown): WorkflowMergeSortRule[] { + const rules = Array.isArray(value) ? value.filter(isRecord).slice(0, 3) : []; + return [0, 1, 2].map((index) => { + const rule = rules[index]; + return { + column: typeof rule?.column === 'string' ? rule.column : '', + descending: rule?.descending === true, + value_type: ['text', 'number', 'date'].includes(String(rule?.value_type)) + ? rule?.value_type : 'text', + } as WorkflowMergeSortRule; + }); +} + +function updateSortRule(options: WorkflowMergeOptions, index: number, patch: Partial): WorkflowMergeOptions { + const rules = sortRules(options.sort_by).map((rule, position) => (position === index ? { ...rule, ...patch } : rule)) + .filter((rule) => rule.column.trim()); + const next = { ...options }; + if (rules.length) next.sort_by = rules; + else delete next.sort_by; + return next; +} + +function moveReference(references: WorkflowReferenceInput[], index: number, direction: -1 | 1): WorkflowReferenceInput[] { + const next = references.slice(); + const target = index + direction; + if (target < 0 || target >= references.length) { + return references; + } + [next[index], next[target]] = [next[target], next[index]]; + return next; +} + +function MergeActionFields({ + action, + evidence, + changedFileTargets, + maxSelectedDocuments, + onActionChange, + onEvidenceChange, +}: { + action: WorkflowDocumentAction | undefined; + evidence: WorkflowReferenceInput[]; + changedFileTargets: boolean; + maxSelectedDocuments?: number; + onActionChange: (updates: { + mergeKind?: WorkflowMergeKind; + targetMode?: WorkflowMergeTargetMode; + outputFormat?: WorkflowMergeOutputFormat; + recentWindowMinutes?: number; + outputFileName?: string; + mergeOptions?: WorkflowMergeOptions; + }) => void; + onEvidenceChange: (references: WorkflowReferenceInput[]) => void; +}) { + const kind = mergeKind(action); + const targetMode = mergeTargetMode(action); + const options = mergeOptions(action); + const [aliasesText, setAliasesText] = useState(() => formatWorkflowMergeColumnAliases(options.column_aliases)); + const aliasParse = parseWorkflowMergeColumnAliases(aliasesText); + const validationErrors = workflowMergeValidationErrors(action, { + isFileSyncWorkflow: changedFileTargets, + maxSelectedDocuments, + }); + const kindOptions = WORKFLOW_MERGE_KINDS_AVAILABLE.includes(kind) + ? WORKFLOW_MERGE_KINDS_AVAILABLE + : [kind, ...WORKFLOW_MERGE_KINDS_AVAILABLE]; + const applyOptions = (nextOptions: WorkflowMergeOptions) => { + onActionChange({ mergeOptions: cleanupWorkflowMergeOptions(kind, nextOptions) ?? nextOptions }); + }; + const applyAliases = (value: string) => { + setAliasesText(value); + const parsed = parseWorkflowMergeColumnAliases(value); + if (!parsed.errors.length) { + const next = { ...options }; + if (Object.keys(parsed.aliases).length) next.column_aliases = parsed.aliases; + else delete next.column_aliases; + applyOptions(next); + } + }; + const sheetMode = options.sheet ? 'named' : options.sheets === 'all' ? 'all' : 'first'; + return ( +
    +
    + + +
    + {targetMode === 'recent' ? ( + + ) : null} +
    + {kind === 'tabular' ? ( + + ) : ( +

    + Output format: {mergeOutputFormat(action, kind).toUpperCase()} +

    + )} + +
    + {targetMode === 'selected' && evidence.length ? ( +
    +

    Merge order

    +
      + {evidence.map((item, index) => ( +
    1. + {index + 1}. {item.name} + onEvidenceChange(moveReference(evidence, index, -1))} + aria-label={`Move ${item.name} up in merge order`}> + Up + + onEvidenceChange(moveReference(evidence, index, 1))} + aria-label={`Move ${item.name} down in merge order`}> + Down + +
      2. + ))} +
    +
    + ) : null} +
    + More merge options +
    + {kind === 'tabular' ? ( + <> + + {options.schema_policy === 'mapped' ? ( +