From 18fc08ac9dc21066a324afbdb499de523717be65 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Mon, 21 Sep 2026 12:13:13 -0400 Subject: [PATCH 1/2] Add orchestration result foundation contracts and facade Implement M0/M1 typed named outputs, strict completeness and bindings, authorized private retained-result readers, and lifecycle-fenced commits. Preserve production v1 execution, saved Analyze, workflows, and exact_records_v1 JSON behavior. Document foundation-only scope at version 0.261.125. Refs #1509 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- ...unctions_orchestration_result_contracts.py | 521 ++++++++++++ .../functions_orchestration_results.py | 787 ++++++++++++++++++ .../functions_workflow_result_store.py | 41 + docs/explanation/features/ANALYZE_RESULTS.md | 10 + .../features/CHAT_ORCHESTRATION.md | 8 +- .../ORCHESTRATION_CHECKPOINT_RECOVERY.md | 9 +- .../ORCHESTRATION_RENDERING_HARNESS.md | 455 ++++++++++ .../test_orchestration_result_contracts.py | 313 +++++++ .../test_orchestration_result_imports.py | 161 ++++ .../test_orchestration_results.py | 547 ++++++++++++ .../test_support/orchestration_results.py | 168 ++++ 12 files changed, 3019 insertions(+), 3 deletions(-) create mode 100644 application/single_app/functions_orchestration_result_contracts.py create mode 100644 application/single_app/functions_orchestration_results.py create mode 100644 docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md create mode 100644 functional_tests/test_orchestration_result_contracts.py create mode 100644 functional_tests/test_orchestration_result_imports.py create mode 100644 functional_tests/test_orchestration_results.py create mode 100644 functional_tests/test_support/orchestration_results.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 9a2c75504..fcd22ca81 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -98,7 +98,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.124" +VERSION = "0.261.125" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_orchestration_result_contracts.py b/application/single_app/functions_orchestration_result_contracts.py new file mode 100644 index 000000000..7c9db570a --- /dev/null +++ b/application/single_app/functions_orchestration_result_contracts.py @@ -0,0 +1,521 @@ +# functions_orchestration_result_contracts.py +"""Opt-in retained-result contracts, independent of application startup and v1 plans.""" + +import hashlib +import json +import math +import re +from dataclasses import dataclass, fields +from typing import ClassVar + + +TASK_RESULT_VERSION = "orchestration-task-result-v1" +RESULT_REF_VERSION = "orchestration-result-ref-v1" +INPUT_BINDING_VERSION = "orchestration-input-binding-v1" +COMPLETENESS_VERSION = "orchestration-completeness-v1" +RESULT_KINDS = frozenset({ + "records-v1", "text-v1", "markdown-v1", "structured-v1", + "evidence-set-v1", "source-set-v1", "comparison-v1", +}) +RESULT_STATES = frozenset({"pending", "partial", "complete", "invalid", "cancelled", "failed", "unavailable"}) +RESULT_ROLES = frozenset({"gather", "reason", "render"}) +REASON_CAPABILITIES = frozenset({"document_analyze", "document_compare", "tabular_analyze"}) +MAX_OUTPUTS = 32 +MAX_COLUMNS = 256 +MAX_BINDING_STEPS = 64 +MAX_DESCRIPTOR_BYTES = 128 * 1024 +_NAME = re.compile(r"[A-Za-z][A-Za-z0-9_-]{0,63}\Z") +_DIGEST = re.compile(r"[a-f0-9]{64}\Z") +_COLUMN_TYPES = frozenset({"string", "integer", "number", "boolean", "object", "array", "json"}) + + +class ResultContractError(ValueError): + """Invalid data is never repaired into a different successful result.""" + + def __init__(self, code="result_contract_invalid"): + self.code = code + super().__init__("The orchestration result or input binding is invalid.") + + +class ResultNotReadyError(ResultContractError): + def __init__(self): + super().__init__("result_not_ready") + + +def identifier(value, *, limit=1024): + if type(value) is not str or not value.strip() or value != value.strip(): + raise ResultContractError("result_identity_invalid") + try: + size = len(value.encode("utf-8")) + except UnicodeError as exc: + raise ResultContractError("result_identity_invalid") from exc + if size > limit: + raise ResultContractError("result_identity_invalid") + return value + + +def output_name(value): + if type(value) is not str or _NAME.fullmatch(value) is None: + raise ResultContractError("result_name_invalid") + return value + + +def digest(value): + if type(value) is not str or _DIGEST.fullmatch(value) is None: + raise ResultContractError("result_digest_invalid") + return value + + +def integer(value, *, minimum=0): + if type(value) is not int or not minimum <= value < 2 ** 63: + raise ResultContractError("result_count_invalid") + return value + + +def choice(value, options): + if type(value) is not str or value not in options: + raise ResultContractError() + return value + + +def validate_json(value, depth=0): + if depth > 64: + raise ResultContractError("result_json_invalid") + if value is None or type(value) in (str, bool, int): + if type(value) is str: + try: + value.encode("utf-8") + except UnicodeError as exc: + raise ResultContractError("result_json_invalid") from exc + return + if type(value) is float and math.isfinite(value): + return + if type(value) is list: + for child in value: + validate_json(child, depth + 1) + return + if type(value) is dict and all(type(key) is str for key in value): + for key, child in value.items(): + validate_json(key, depth + 1) + validate_json(child, depth + 1) + return + raise ResultContractError("result_json_invalid") + + +def canonical_bytes(value): + validate_json(value) + return json.dumps( + value, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":"), + ).encode("ascii") + + +def canonical_digest(value): + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def _tuple(value, item_type, *, maximum): + if type(value) is not tuple or len(value) > maximum or any(type(item) is not item_type for item in value): + raise ResultContractError() + + +def _list(value): + if type(value) is not list: + raise ResultContractError() + return value + + +def _wire(value): + if isinstance(value, _Contract): + return value.to_dict() + if type(value) is tuple: + return [_wire(item) for item in value] + return value + + +class _Contract: + VERSION: ClassVar[str | None] = None + + def to_dict(self): + value = {field.name: _wire(getattr(self, field.name)) for field in fields(self)} + if self.VERSION is not None: + value = {"version": self.VERSION, **value} + return value + + @classmethod + def _values(cls, value): + expected = {field.name for field in fields(cls)} + if cls.VERSION is not None: + expected.add("version") + if type(value) is not dict or set(value) != expected: + raise ResultContractError() + if cls.VERSION is not None and value["version"] != cls.VERSION: + raise ResultContractError("result_version_unsupported") + return {key: item for key, item in value.items() if key != "version"} + + +@dataclass(frozen=True) +class ProducerIdentity(_Contract): + user_id: str + conversation_id: str + run_id: str + attempt_index: int + step_id: str + capability_id: str + contract_version: str + + def __post_init__(self): + for name in ("user_id", "conversation_id", "run_id", "step_id", "capability_id", "contract_version"): + identifier(getattr(self, name)) + integer(self.attempt_index, minimum=1) + + @classmethod + def from_dict(cls, value): + return cls(**cls._values(value)) + + +def validate_result_role(producer, role): + if type(producer) is not ProducerIdentity: + raise ResultContractError("result_producer_invalid") + choice(role, RESULT_ROLES) + if producer.capability_id in REASON_CAPABILITIES and role != "reason": + raise ResultContractError("result_role_mismatch") + + +@dataclass(frozen=True) +class Coverage(_Contract): + expected: int | None + completed: int + unit: str + + def __post_init__(self): + integer(self.completed) + if self.expected is not None: + integer(self.expected) + if self.completed > self.expected: + raise ResultContractError("result_coverage_invalid") + choice(self.unit, {"sources", "work_units", "records", "items"}) + + @classmethod + def from_dict(cls, value): + return cls(**cls._values(value)) + + +@dataclass(frozen=True) +class Completeness(_Contract): + status: str + expected_count: int | None + actual_count: int + coverage: Coverage + validation: str + checks: tuple[str, ...] + limitations: tuple[str, ...] + preview: bool = False + VERSION = COMPLETENESS_VERSION + + def __post_init__(self): + choice(self.status, RESULT_STATES) + choice(self.validation, {"valid", "partial", "invalid", "pending", "not_validated"}) + integer(self.actual_count) + if self.expected_count is not None: + integer(self.expected_count) + if self.actual_count > self.expected_count: + raise ResultContractError("result_count_invalid") + if type(self.coverage) is not Coverage or type(self.preview) is not bool: + raise ResultContractError() + for value in (self.checks, self.limitations): + _tuple(value, str, maximum=32) + for item in value: + identifier(item, limit=1024) + if self.status == "complete" and ( + self.preview or self.validation != "valid" or not self.checks + or self.expected_count != self.actual_count or self.coverage.expected is None + or self.coverage.expected != self.coverage.completed + ): + raise ResultContractError("result_incomplete") + if self.status == "partial" and ( + self.preview or self.validation not in {"valid", "partial"} or not self.limitations + ): + raise ResultContractError("result_partial_unqualified") + + def require_readable(self, *, allow_partial=False): + if type(allow_partial) is not bool: + raise ResultContractError() + if self.preview or self.status not in ({"complete", "partial"} if allow_partial else {"complete"}): + raise ResultNotReadyError() + + @classmethod + def from_dict(cls, value): + values = cls._values(value) + values["coverage"] = Coverage.from_dict(values["coverage"]) + values["checks"] = tuple(_list(values["checks"])) + values["limitations"] = tuple(_list(values["limitations"])) + return cls(**values) + + +@dataclass(frozen=True) +class RecordColumn(_Contract): + name: str + value_type: str + nullable: bool = False + + def __post_init__(self): + identifier(self.name, limit=256) + choice(self.value_type, _COLUMN_TYPES) + if type(self.nullable) is not bool: + raise ResultContractError("result_schema_invalid") + + @classmethod + def from_dict(cls, value): + return cls(**cls._values(value)) + + +def validate_columns(columns, kind): + _tuple(columns, RecordColumn, maximum=MAX_COLUMNS) + if (kind == "records-v1" and not columns) or (kind != "records-v1" and columns): + raise ResultContractError("result_schema_invalid") + if len({column.name for column in columns}) != len(columns): + raise ResultContractError("result_schema_invalid") + + +def validate_record(record, columns): + if type(record) is not dict or set(record) != {column.name for column in columns}: + raise ResultContractError("result_schema_invalid") + validate_json(record) + types = { + "string": (str,), "integer": (int,), "number": (int, float), + "boolean": (bool,), "object": (dict,), "array": (list,), + "json": (str, int, float, bool, dict, list), + } + for column in columns: + value = record[column.name] + if value is None: + if not column.nullable: + raise ResultContractError("result_schema_invalid") + elif type(value) not in types[column.value_type]: + raise ResultContractError("result_schema_invalid") + + +@dataclass(frozen=True) +class ResultRef(_Contract): + producer: ProducerIdentity + output_name: str + kind: str + manifest_sha256: str + content_sha256: str + size_bytes: int + item_count: int + completeness: Completeness + columns: tuple[RecordColumn, ...] = () + character_count: int | None = None + VERSION = RESULT_REF_VERSION + + def __post_init__(self): + if type(self.producer) is not ProducerIdentity or type(self.completeness) is not Completeness: + raise ResultContractError() + output_name(self.output_name) + choice(self.kind, RESULT_KINDS) + digest(self.manifest_sha256) + digest(self.content_sha256) + integer(self.size_bytes) + integer(self.item_count) + if self.completeness.actual_count != self.item_count: + raise ResultContractError("result_count_invalid") + validate_columns(self.columns, self.kind) + if self.kind in {"text-v1", "markdown-v1"}: + integer(self.character_count) + if not self.character_count <= self.size_bytes <= 4 * self.character_count: + raise ResultContractError("result_count_invalid") + elif self.character_count is not None: + raise ResultContractError("result_count_invalid") + if len(canonical_bytes(self.to_dict())) > MAX_DESCRIPTOR_BYTES: + raise ResultContractError("result_descriptor_too_large") + + @classmethod + def from_dict(cls, value): + values = cls._values(value) + values["producer"] = ProducerIdentity.from_dict(values["producer"]) + values["completeness"] = Completeness.from_dict(values["completeness"]) + values["columns"] = tuple(RecordColumn.from_dict(item) for item in _list(values["columns"])) + return cls(**values) + + +@dataclass(frozen=True) +class TaskResult(_Contract): + producer: ProducerIdentity + role: str + status: str + outputs: tuple[ResultRef, ...] + VERSION = TASK_RESULT_VERSION + + def __post_init__(self): + if type(self.producer) is not ProducerIdentity: + raise ResultContractError() + validate_result_role(self.producer, self.role) + choice(self.status, RESULT_STATES) + _tuple(self.outputs, ResultRef, maximum=MAX_OUTPUTS) + if any(reference.producer != self.producer for reference in self.outputs): + raise ResultContractError("result_producer_mismatch") + if len({reference.output_name for reference in self.outputs}) != len(self.outputs): + raise ResultContractError("result_duplicate_output") + if len({reference.manifest_sha256 for reference in self.outputs}) > 1: + raise ResultContractError("result_manifest_mismatch") + if self.status == "complete" and ( + not self.outputs or any(reference.completeness.status != "complete" for reference in self.outputs) + ): + raise ResultContractError("result_incomplete") + if len(canonical_bytes(self.to_dict())) > MAX_DESCRIPTOR_BYTES: + raise ResultContractError("result_descriptor_too_large") + + def output(self, name): + output_name(name) + for reference in self.outputs: + if reference.output_name == name: + return reference + raise ResultContractError("result_output_missing") + + @classmethod + def from_dict(cls, value): + values = cls._values(value) + values["producer"] = ProducerIdentity.from_dict(values["producer"]) + values["outputs"] = tuple(ResultRef.from_dict(item) for item in _list(values["outputs"])) + return cls(**values) + + +@dataclass(frozen=True) +class InputBinding(_Contract): + step_id: str | None = None + output_name: str | None = None + existing_result: str | None = None + VERSION = INPUT_BINDING_VERSION + + def __post_init__(self): + if self.existing_result is not None: + output_name(self.existing_result) + if self.step_id is not None or self.output_name is not None: + raise ResultContractError("result_binding_invalid") + else: + identifier(self.step_id) + output_name(self.output_name) + + @classmethod + def from_dict(cls, value): + return cls(**cls._values(value)) + + +@dataclass(frozen=True) +class InputSpec: + name: str + binding: InputBinding + kinds: tuple[str, ...] + allow_partial: bool = False + + def __post_init__(self): + output_name(self.name) + if type(self.binding) is not InputBinding or type(self.allow_partial) is not bool: + raise ResultContractError() + _tuple(self.kinds, str, maximum=len(RESULT_KINDS)) + if not self.kinds or len(set(self.kinds)) != len(self.kinds): + raise ResultContractError() + for kind in self.kinds: + choice(kind, RESULT_KINDS) + + +@dataclass(frozen=True) +class OutputSpec: + name: str + kind: str + + def __post_init__(self): + output_name(self.name) + choice(self.kind, RESULT_KINDS) + + +@dataclass(frozen=True) +class StepBindings: + step_id: str + enabled: bool + outputs: tuple[OutputSpec, ...] = () + inputs: tuple[InputSpec, ...] = () + depends_on: tuple[str, ...] = () + + def __post_init__(self): + identifier(self.step_id) + if type(self.enabled) is not bool: + raise ResultContractError() + _tuple(self.outputs, OutputSpec, maximum=MAX_OUTPUTS) + _tuple(self.inputs, InputSpec, maximum=MAX_OUTPUTS) + _tuple(self.depends_on, str, maximum=MAX_BINDING_STEPS) + for collection in (self.outputs, self.inputs): + if len({item.name for item in collection}) != len(collection): + raise ResultContractError("result_duplicate_binding") + for step_id in self.depends_on: + identifier(step_id) + if len(set(self.depends_on)) != len(self.depends_on): + raise ResultContractError("result_duplicate_dependency") + + +def validate_input_bindings(steps, *, existing_results=None, max_steps=MAX_BINDING_STEPS): + """Return inferred dependencies without modifying, sorting, or repairing a v1 plan. + + Input/output specifications and the existing-result alias catalog are supplied + by the server, not by a model. Resolving an alias still requires current access. + """ + integer(max_steps, minimum=1) + if max_steps > MAX_BINDING_STEPS or type(steps) not in (list, tuple) or len(steps) > max_steps: + raise ResultContractError("result_step_limit") + if any(type(step) is not StepBindings for step in steps): + raise ResultContractError() + by_id = {step.step_id: step for step in steps} + if len(by_id) != len(steps): + raise ResultContractError("result_duplicate_step") + catalog = {} if existing_results is None else existing_results + if type(catalog) is not dict: + raise ResultContractError() + for alias, reference in catalog.items(): + output_name(alias) + if type(reference) is not ResultRef: + raise ResultContractError("result_reference_untrusted") + dependencies = {} + for step in steps: + required = list(step.depends_on) + for spec in step.inputs: + binding = spec.binding + if binding.existing_result is not None: + reference = catalog.get(binding.existing_result) + if reference is None: + raise ResultContractError("result_reference_untrusted") + reference.completeness.require_readable(allow_partial=spec.allow_partial) + kind = reference.kind + else: + producer = by_id.get(binding.step_id) + if producer is None: + raise ResultContractError("result_producer_missing") + kind = next((output.kind for output in producer.outputs if output.name == binding.output_name), None) + if kind is None: + raise ResultContractError("result_output_missing") + if binding.step_id not in required: + required.append(binding.step_id) + if kind not in spec.kinds: + raise ResultContractError("result_kind_incompatible") + for dependency in required: + if dependency not in by_id: + raise ResultContractError("result_producer_missing") + if not by_id[dependency].enabled: + raise ResultContractError("result_producer_disabled") + dependencies[step.step_id] = tuple(required) + visiting, visited = set(), set() + + def visit(step_id): + if step_id in visiting: + raise ResultContractError("result_binding_cycle") + if step_id in visited: + return + visiting.add(step_id) + for dependency in dependencies[step_id]: + visit(dependency) + visiting.remove(step_id) + visited.add(step_id) + + for step in steps: + visit(step.step_id) + return dependencies diff --git a/application/single_app/functions_orchestration_results.py b/application/single_app/functions_orchestration_results.py new file mode 100644 index 000000000..1051ba6f4 --- /dev/null +++ b/application/single_app/functions_orchestration_results.py @@ -0,0 +1,787 @@ +# functions_orchestration_results.py +"""Authorized retained results for future orchestration adapters, not a new runtime. + +Owners inject initialized storage and current access readers. No renderer, route, +settings owner, model, artifact publisher, or global result catalog is imported. +""" + +import hashlib +import json +from collections.abc import Iterable +from copy import deepcopy +from dataclasses import dataclass + +from content_screening.access import assert_evidence_available +from functions_analysis_access import analysis_source_snapshot, authorize_analysis_sources +from functions_orchestration_result_contracts import ( + MAX_DESCRIPTOR_BYTES, + MAX_OUTPUTS, + RESULT_KINDS, + RESULT_STATES, + TASK_RESULT_VERSION, + Completeness, + InputSpec, + ProducerIdentity, + RecordColumn, + ResultContractError, + ResultRef, + TaskResult, + canonical_bytes, + choice, + identifier, + integer, + output_name, + validate_columns, + validate_json, + validate_record, + validate_result_role, +) +from functions_workflow_collections import ( + CollectionWriteBudget, + RecordTreeWriter, + iter_record_tree, +) +from functions_workflow_results import read_result_records + + +MAX_VALUE_BYTES = 8 * 1024 * 1024 +MAX_PREVIEW_BYTES = 16 * 1024 +MAX_PREVIEW_ITEMS = 20 +MAX_UPSTREAM_RESULTS = 64 +MAX_LINEAGE_RESULTS = 256 +_COLLECTION_KINDS = frozenset({"records-v1", "evidence-set-v1", "source-set-v1"}) +_TEXT_KINDS = frozenset({"text-v1", "markdown-v1"}) +_MANIFEST_FIELDS = frozenset({"version", "producer", "role", "status", "lineage", "outputs"}) +_OUTPUT_FIELDS = frozenset({ + "kind", "columns", "completeness", "content_sha256", "size_bytes", "item_count", "character_count", "storage", +}) +_LINEAGE_FIELDS = frozenset({"origin", "source_policy", "sources", "upstream", "allow_partial_inputs"}) + + +class ResultUnavailableError(PermissionError): + def __init__(self, code="result_unavailable"): + self.code = code + super().__init__("This retained result is unavailable under the current access or producer state.") + + +@dataclass(frozen=True) +class NamedOutput: + name: str + kind: str + value: object + completeness: Completeness + columns: tuple[RecordColumn, ...] = () + + def __post_init__(self): + output_name(self.name) + choice(self.kind, RESULT_KINDS) + if type(self.completeness) is not Completeness or self.completeness.preview: + raise ResultContractError("result_preview_not_authoritative") + validate_columns(self.columns, self.kind) + + +class OrchestrationResultAccess: + """Server-owned owner/run readers and authorized, screening-aware source I/O. + + ``read_conversation(conversation_id)`` and ``read_run(run_id)`` return current + server records, not request payloads. ``source_resolver(ids, **scope)`` has the + existing resolve_authorized_source_manifest protocol. ``source_metadata_reader`` + has content_screening.access's metadata_reader protocol and must enforce the + current actor's document/workspace access. Source-free results need neither + source callback. Original-owner, same-conversation reuse is the v1 scope. + """ + + def __init__( + self, *, user_id, conversation_id, read_conversation, read_run, + source_resolver=None, source_metadata_reader=None, + ): + self.user_id = identifier(user_id) + self.conversation_id = identifier(conversation_id) + if not callable(read_conversation) or not callable(read_run): + raise ResultContractError("result_access_reader_required") + for callback in (source_resolver, source_metadata_reader): + if callback is not None and not callable(callback): + raise ResultContractError("result_access_reader_required") + self.read_conversation = read_conversation + self.read_run = read_run + self.source_resolver = source_resolver + self.source_metadata_reader = source_metadata_reader + + def authorize_producer(self, producer, *, for_write=False): + if ( + type(producer) is not ProducerIdentity or producer.user_id != self.user_id + or producer.conversation_id != self.conversation_id + ): + raise ResultUnavailableError("result_owner_mismatch") + conversation = self.read_conversation(self.conversation_id) + if ( + type(conversation) is not dict or conversation.get("id") != self.conversation_id + or conversation.get("user_id") != self.user_id or conversation.get("orchestration_deleted") + ): + raise ResultUnavailableError("result_conversation_unavailable") + run = self.read_run(producer.run_id) + if ( + type(run) is not dict or run.get("id") != producer.run_id + or run.get("user_id") != self.user_id or run.get("conversation_id") != self.conversation_id + or run.get("checkpoints_deleted") + ): + raise ResultUnavailableError("result_producer_unavailable") + # Historical v1 runs predate the explicit index; their original attempt is 1. + attempt = run.get("attempt_index", 1) + if type(attempt) is not int or attempt != producer.attempt_index: + raise ResultUnavailableError("result_attempt_mismatch") + plan = run.get("plan") + steps = plan.get("steps") if type(plan) is dict else None + if type(steps) is not list or any(type(step) is not dict for step in steps): + raise ResultUnavailableError("result_producer_unavailable") + matching = [step for step in steps if step.get("step_id") == producer.step_id] + if ( + len(matching) != 1 or matching[0].get("enabled", True) is not True + or matching[0].get("capability_id") != producer.capability_id + ): + raise ResultUnavailableError("result_producer_unavailable") + if for_write and ( + run.get("status") != "running" or run.get("cancellation_requested_at") + or run.get("latest_attempt_run_id") + ): + raise ResultUnavailableError("result_attempt_stopped") + + def authorize_sources(self, sources, *, require_snapshot): + if not sources: + return {"source_count": 0, "source_snapshot_changed": False} + if self.source_resolver is None or self.source_metadata_reader is None: + raise ResultUnavailableError("result_source_reader_required") + originals = {_source_key(source): source for source in sources} + digest_changed = False + + def resolve(document_ids, **scope): + nonlocal digest_changed + fresh = self.source_resolver(document_ids, **scope) + if type(fresh) is list: + for source in fresh: + if type(source) is not dict or not {"document_id", "scope", "scope_id"}.issubset(source): + continue + original = originals.get(_source_key(source)) or {} + expected = original.get("content_sha256") + if expected is not None and source.get("content_sha256") != expected: + digest_changed = True + if require_snapshot: + raise ResultUnavailableError("result_source_snapshot_changed") + return fresh + + checked = authorize_analysis_sources( + self.user_id, sources, require_snapshot=require_snapshot, resolver=resolve, + ) + assert_evidence_available( + sources, user_id=self.user_id, metadata_reader=self.source_metadata_reader, + ) + return {**checked, "source_snapshot_changed": checked["source_snapshot_changed"] or digest_changed} + + +def _producer_arguments(producer): + return (producer.user_id, producer.conversation_id, producer.run_id, producer.step_id) + + +def _source_key(source): + return source["document_id"], source["scope"], source["scope_id"] + + +def _sources(value): + snapshots = analysis_source_snapshot(value) + if snapshots != value: + raise ResultContractError("result_source_snapshot_invalid") + keys = [_source_key(source) for source in snapshots] + if len(set(keys)) != len(keys): + raise ResultContractError("result_source_snapshot_invalid") + return snapshots + + +def _lineage(value): + if type(value) is not dict or set(value) != _LINEAGE_FIELDS: + raise ResultContractError("result_lineage_invalid") + choice(value["origin"], {"generated", "grounded"}) + choice(value["source_policy"], {"current", "snapshot"}) + _sources(value["sources"]) + if type(value["allow_partial_inputs"]) is not bool: + raise ResultContractError("result_lineage_invalid") + parents = value["upstream"] + if type(parents) is not list or len(parents) > MAX_UPSTREAM_RESULTS: + raise ResultContractError("result_lineage_invalid") + references = tuple(ResultRef.from_dict(item) for item in parents) + if len(set(references)) != len(references): + raise ResultContractError("result_lineage_invalid") + if (value["origin"] == "generated") != (not value["sources"] and not references): + raise ResultContractError("result_lineage_invalid") + if len(canonical_bytes(value)) > MAX_DESCRIPTOR_BYTES: + raise ResultContractError("result_descriptor_too_large") + return references + + +def _reference(producer, name, output, manifest_sha256): + if type(output) is not dict or set(output) != _OUTPUT_FIELDS or type(output["columns"]) is not list: + raise ResultContractError("result_manifest_invalid") + reference = ResultRef( + producer=producer, output_name=name, kind=output["kind"], manifest_sha256=manifest_sha256, + content_sha256=output["content_sha256"], size_bytes=output["size_bytes"], + item_count=output["item_count"], completeness=Completeness.from_dict(output["completeness"]), + columns=tuple(RecordColumn.from_dict(item) for item in output["columns"]), + character_count=output["character_count"], + ) + storage = output["storage"] + if ( + type(storage) is not dict or set(storage) != {"kind", "storage_kind", "result_ref", "record_count"} + or storage["kind"] != "records" or storage["storage_kind"] != "record_tree" + ): + raise ResultContractError("result_manifest_invalid") + integer(storage["record_count"]) + if reference.kind in _COLLECTION_KINDS and storage["record_count"] != reference.item_count: + raise ResultContractError("result_count_invalid") + if reference.kind in _TEXT_KINDS | {"structured-v1"} and reference.item_count != 1: + raise ResultContractError("result_count_invalid") + if reference.kind == "comparison-v1" and reference.size_bytes > MAX_VALUE_BYTES: + raise ResultContractError("result_comparison_too_large") + return reference + + +def _task_result(manifest, manifest_sha256): + if type(manifest) is not dict or set(manifest) != _MANIFEST_FIELDS or manifest["version"] != TASK_RESULT_VERSION: + raise ResultContractError("result_manifest_invalid") + if len(canonical_bytes(manifest)) > MAX_DESCRIPTOR_BYTES: + raise ResultContractError("result_descriptor_too_large") + producer = ProducerIdentity.from_dict(manifest["producer"]) + _lineage(manifest["lineage"]) + outputs = manifest["outputs"] + if type(outputs) is not dict or not 1 <= len(outputs) <= MAX_OUTPUTS: + raise ResultContractError("result_manifest_invalid") + return TaskResult( + producer, manifest["role"], manifest["status"], + tuple(_reference(producer, name, output, manifest_sha256) for name, output in outputs.items()), + ) + + +def _collection_manifest(producer, name, storage): + return {"contract_version": TASK_RESULT_VERSION, "identity": producer.to_dict(), "outputs": {name: storage}} + + +def _validate_item(kind, item, columns, source_snapshots): + validate_json(item) + if kind == "records-v1": + validate_record(item, columns) + elif kind == "source-set-v1": + _sources([item]) + if canonical_bytes(item) not in source_snapshots.get(_source_key(item), ()): + raise ResultContractError("result_source_not_in_lineage") + elif kind == "evidence-set-v1": + if type(item) is not dict or set(item) != {"evidence_id", "source", "text"}: + raise ResultContractError("result_evidence_invalid") + identifier(item["evidence_id"]) + if type(item["text"]) is not str: + raise ResultContractError("result_evidence_invalid") + _sources([item["source"]]) + if canonical_bytes(item["source"]) not in source_snapshots.get(_source_key(item["source"]), ()): + raise ResultContractError("result_source_not_in_lineage") + + +def _validate_value(kind, value, completeness, source_snapshots): + validate_json(value) + if kind in _TEXT_KINDS: + if type(value) is not str: + raise ResultContractError("result_value_invalid") + count = 1 + elif kind == "comparison-v1": + expected_fields = {"left_document_id", "right_document_ids", "items", "failed_document_ids"} + if type(value) is not dict or set(value) != expected_fields: + raise ResultContractError("result_comparison_invalid") + identifier(value["left_document_id"]) + for name in ("right_document_ids", "failed_document_ids", "items"): + if type(value[name]) is not list: + raise ResultContractError("result_comparison_invalid") + targets = value["right_document_ids"] + failed = value["failed_document_ids"] + for document_id in targets + failed: + identifier(document_id) + completed = [] + for item in value["items"]: + if type(item) is not dict or set(item) != {"right_document_id", "right_document_name", "text"}: + raise ResultContractError("result_comparison_invalid") + identifier(item["right_document_id"]) + identifier(item["right_document_name"]) + identifier(item["text"], limit=MAX_VALUE_BYTES) + completed.append(item["right_document_id"]) + if ( + not targets or value["left_document_id"] in targets or len(set(targets)) != len(targets) + or len(set(failed + completed)) != len(failed + completed) or set(failed + completed) != set(targets) + or not set([value["left_document_id"], *targets]).issubset({key[0] for key in source_snapshots}) + or completeness.expected_count != len(targets) + or (completeness.status == "complete" and failed) + ): + raise ResultContractError("result_comparison_incomplete") + count = len(completed) + else: + count = 1 + if completeness.actual_count != count: + raise ResultContractError("result_count_invalid") + return count + + +class OrchestrationResults: + """A small facade over an injected WorkflowResultStore, with no client factory.""" + + def __init__(self, store, access, *, max_result_bytes=None): + if type(access) is not OrchestrationResultAccess: + raise ResultContractError("result_access_reader_required") + self.store = store + self.access = access + self.max_result_bytes = store.max_size_bytes if max_result_bytes is None else max_result_bytes + integer(self.max_result_bytes, minimum=1) + + def _load_manifest(self, reference): + self.access.authorize_producer(reference.producer) + manifest = self.store.load_committed_orchestration_result( + *_producer_arguments(reference.producer), reference.manifest_sha256, + ) + task = _task_result(manifest, reference.manifest_sha256) + if task.producer != reference.producer or task.output(reference.output_name) != reference: + raise ResultContractError("result_reference_mismatch") + return manifest + + def _authorize_lineage(self, producer, lineage, *, for_write=False, force_current=False, active=None, visited=None): + self.access.authorize_producer(producer, for_write=for_write) + parents = _lineage(lineage) + current = force_current or lineage["source_policy"] == "current" + checked = self.access.authorize_sources(lineage["sources"], require_snapshot=current) + source_snapshots = { + _source_key(source): {canonical_bytes(source)} for source in lineage["sources"] + } + changed = checked["source_snapshot_changed"] + active = set() if active is None else active + visited = {} if visited is None else visited + for parent in parents: + if parent.producer == producer: + raise ResultContractError("result_lineage_cycle") + parent.completeness.require_readable(allow_partial=lineage["allow_partial_inputs"]) + key = (parent.producer, parent.manifest_sha256, current) + if key in active: + raise ResultContractError("result_lineage_cycle") + if len(active) + len(visited) >= MAX_LINEAGE_RESULTS: + raise ResultContractError("result_lineage_limit") + manifest = self._load_manifest(parent) + if key not in visited: + active.add(key) + try: + visited[key] = self._authorize_lineage( + parent.producer, manifest["lineage"], force_current=current, active=active, visited=visited, + ) + finally: + active.remove(key) + parent_changed, parent_snapshots = visited[key] + changed = changed or parent_changed + for source_key, snapshots in parent_snapshots.items(): + source_snapshots.setdefault(source_key, set()).update(snapshots) + return changed, source_snapshots + + def persist_task_result( + self, *, producer, role, status, outputs, sources, origin, guard_token, + upstream=(), source_policy="current", allow_partial_inputs=False, + ): + """Retain final data/diagnostics and return only immutable named descriptors. + + Collection values can be one-pass iterators. Every write is private and + fenced; only the last, immutable server commit makes a descriptor readable. + A pending/invalid descriptor is retained honestly but is not consumable. + """ + if type(producer) is not ProducerIdentity: + raise ResultContractError("result_producer_invalid") + validate_result_role(producer, role) + choice(status, RESULT_STATES) + identifier(guard_token) + if type(outputs) not in (list, tuple) or not 1 <= len(outputs) <= MAX_OUTPUTS: + raise ResultContractError("result_outputs_invalid") + if any(type(output) is not NamedOutput for output in outputs): + raise ResultContractError("result_outputs_invalid") + if len({output.name for output in outputs}) != len(outputs): + raise ResultContractError("result_duplicate_output") + if type(upstream) not in (list, tuple) or any(type(parent) is not ResultRef for parent in upstream): + raise ResultContractError("result_lineage_invalid") + lineage = { + "origin": origin, "source_policy": source_policy, "sources": deepcopy(sources), + "upstream": [parent.to_dict() for parent in upstream], "allow_partial_inputs": allow_partial_inputs, + } + parents = _lineage(lineage) + if any(parent.completeness.status == "partial" for parent in parents) and any( + output.completeness.status == "complete" for output in outputs + ): + raise ResultContractError("result_partial_promoted") + _, source_snapshots = self._authorize_lineage(producer, lineage, for_write=True) + if status == "complete" and any(output.completeness.status != "complete" for output in outputs): + raise ResultContractError("result_incomplete") + self.store.prepare_orchestration_result(*_producer_arguments(producer), guard_token=guard_token) + budget = CollectionWriteBudget(self.max_result_bytes) + + def save(section): + self._authorize_lineage(producer, lineage, for_write=True) + return self.store.save_orchestration( + *_producer_arguments(producer), section, guard_token=guard_token, require_analysis_guard=True, + ) + + stored_outputs = {} + for output in outputs: + writer = RecordTreeWriter( + producer.to_dict(), output.name, "records", save, + max_result_bytes=self.max_result_bytes, contract_version=TASK_RESULT_VERSION, budget=budget, + ) + content_digest = hashlib.sha256() + size_bytes = 0 + + def account(data): + nonlocal size_bytes + content_digest.update(data) + size_bytes += len(data) + if output.kind == "comparison-v1" and size_bytes > MAX_VALUE_BYTES: + raise ResultContractError("result_comparison_too_large") + + if output.kind in _COLLECTION_KINDS: + if not isinstance(output.value, Iterable) or isinstance(output.value, (str, bytes, dict)): + raise ResultContractError("result_records_invalid") + account(b"[") + count = 0 + identities = set() + for item in output.value: + if count >= output.completeness.actual_count: + raise ResultContractError("result_count_invalid") + encoded = canonical_bytes(item) + item = json.loads(encoded) + _validate_item(output.kind, item, output.columns, source_snapshots) + if output.kind != "records-v1": + key = item["evidence_id"] if output.kind == "evidence-set-v1" else _source_key(item) + if key in identities: + raise ResultContractError("result_duplicate_item") + identities.add(key) + if count: + account(b",") + account(encoded) + writer.append(item) + count += 1 + account(b"]") + if count != output.completeness.actual_count: + raise ResultContractError("result_count_invalid") + else: + count = _validate_value(output.kind, output.value, output.completeness, source_snapshots) + if output.kind in _TEXT_KINDS: + for offset in range(0, len(output.value), 4096): + text = output.value[offset:offset + 4096] + account(text.encode("utf-8")) + writer.append({"text": text}) + else: + encoder = json.JSONEncoder( + ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":"), + ) + for fragment in encoder.iterencode(output.value): + for offset in range(0, len(fragment), 16384): + text = fragment[offset:offset + 16384] + account(text.encode("ascii")) + writer.append({"data": text}) + stored_outputs[output.name] = { + "kind": output.kind, "columns": [column.to_dict() for column in output.columns], + "completeness": output.completeness.to_dict(), "content_sha256": content_digest.hexdigest(), + "size_bytes": size_bytes, "item_count": count, "storage": writer.finish(), + "character_count": len(output.value) if output.kind in _TEXT_KINDS else None, + } + manifest = { + "version": TASK_RESULT_VERSION, "producer": producer.to_dict(), "role": role, "status": status, + "lineage": lineage, "outputs": stored_outputs, + } + encoded = canonical_bytes(manifest) + result = _task_result(manifest, hashlib.sha256(encoded).hexdigest()) + reference = budget.save_section(manifest, save, max_section_bytes=MAX_DESCRIPTOR_BYTES) + if reference["sha256"] != result.outputs[0].manifest_sha256: + raise ResultContractError("result_digest_invalid") + self._authorize_lineage(producer, lineage, for_write=True) + self.store.commit_orchestration_result(*_producer_arguments(producer), reference, guard_token=guard_token) + self._authorize_lineage(producer, lineage, for_write=True) + return result + + def open_result(self, reference, *, allow_partial=False, require_current_sources=False): + if type(reference) is not ResultRef or type(require_current_sources) is not bool: + raise ResultContractError("result_reference_untrusted") + reference.completeness.require_readable(allow_partial=allow_partial) + manifest = self._load_manifest(reference) + self._authorize_lineage(reference.producer, manifest["lineage"], force_current=require_current_sources) + return OrchestrationResultReader( + self, reference, manifest, allow_partial=allow_partial, require_current_sources=require_current_sources, + ) + + def resolve_input(self, spec, *, consumer, task_results, existing_results=None): + """Resolve only server-selected task outputs or admitted same-conversation aliases.""" + if type(spec) is not InputSpec or type(consumer) is not ProducerIdentity or type(task_results) is not dict: + raise ResultContractError("result_binding_invalid") + self.access.authorize_producer(consumer) + binding = spec.binding + if binding.existing_result is not None: + catalog = {} if existing_results is None else existing_results + if type(catalog) is not dict: + raise ResultContractError("result_reference_untrusted") + reference = catalog.get(binding.existing_result) + if type(reference) is not ResultRef: + raise ResultContractError("result_reference_untrusted") + else: + task = task_results.get(binding.step_id) + if type(task) is not TaskResult: + raise ResultContractError("result_producer_missing") + if ( + task.producer.step_id != binding.step_id or binding.step_id == consumer.step_id + or any( + getattr(task.producer, key) != getattr(consumer, key) + for key in ("user_id", "conversation_id", "run_id", "attempt_index") + ) + ): + raise ResultContractError("result_attempt_mismatch") + reference = task.output(binding.output_name) + if reference.producer == consumer: + raise ResultContractError("result_binding_cycle") + if reference.kind not in spec.kinds: + raise ResultContractError("result_kind_incompatible") + return self.open_result(reference, allow_partial=spec.allow_partial) + + +class OrchestrationResultReader: + """Full readers verify the last item, declared counts, and the complete digest. + + ``kind == 'records'``, ``record_count``, ``iter_records()`` and ``recheck()`` + implement GeneratedRecordExportSource without importing a renderer. + """ + + def __init__(self, service, reference, manifest, *, allow_partial, require_current_sources): + self._service = service + self.reference = reference + self._manifest = deepcopy(manifest) + self._allow_partial = allow_partial + self._require_current_sources = require_current_sources + self.result_kind = reference.kind + self.kind = "records" if reference.kind == "records-v1" else reference.kind + self.record_count = reference.item_count if self.kind == "records" else None + self.character_count = reference.character_count + self.columns = reference.columns + self.completeness = reference.completeness + self._source_snapshots = {} + self._changed = False + + def recheck(self): + self.reference.completeness.require_readable(allow_partial=self._allow_partial) + self._service._load_manifest(self.reference) + self._changed, self._source_snapshots = self._service._authorize_lineage( + self.reference.producer, self._manifest["lineage"], force_current=self._require_current_sources, + ) + + def metadata(self): + self.recheck() + return { + "reference": self.reference.to_dict(), "source_count": len(self._source_snapshots), + "source_snapshot_changed": self._changed, "origin": self._manifest["lineage"]["origin"], + "source_policy": self._manifest["lineage"]["source_policy"], + } + + def _collection(self): + output = self._manifest["outputs"][self.reference.output_name] + return _collection_manifest(self.reference.producer, self.reference.output_name, output["storage"]) + + def _load_section(self, reference): + self.recheck() + if type(reference) is not dict or type(reference.get("size_bytes")) is not int: + raise ResultContractError("result_reference_untrusted") + if reference["size_bytes"] > MAX_VALUE_BYTES: + raise ResultContractError("result_section_too_large") + return self._service.store.load_orchestration(*_producer_arguments(self.reference.producer), reference) + + def _items(self): + self.recheck() + yield from iter_record_tree(self._collection(), self.reference.output_name, self._load_section) + + def iter_items(self): + if self.result_kind not in _COLLECTION_KINDS: + raise ResultContractError("result_kind_incompatible") + content_digest, size, count = hashlib.sha256(b"["), 1, 0 + identities = set() + for item in self._items(): + _validate_item(self.result_kind, item, self.columns, self._source_snapshots) + if self.result_kind != "records-v1": + key = item["evidence_id"] if self.result_kind == "evidence-set-v1" else _source_key(item) + if key in identities: + raise ResultContractError("result_duplicate_item") + identities.add(key) + encoded = (b"," if count else b"") + canonical_bytes(item) + content_digest.update(encoded) + size += len(encoded) + count += 1 + if count > self.reference.item_count: + raise ResultContractError("result_content_integrity") + yield deepcopy(item) + content_digest.update(b"]") + self._verify(content_digest, size + 1, count) + + def iter_records(self): + if self.result_kind != "records-v1": + raise ResultContractError("result_kind_incompatible") + for record in self.iter_items(): + yield {column.name: record[column.name] for column in self.columns} + + def _verify(self, content_digest, size, count): + if ( + content_digest.hexdigest() != self.reference.content_sha256 + or size != self.reference.size_bytes or count != self.reference.item_count + ): + raise ResultContractError("result_content_integrity") + self.recheck() + + def iter_text(self): + if self.result_kind not in _TEXT_KINDS: + raise ResultContractError("result_kind_incompatible") + content_digest, size, characters = hashlib.sha256(), 0, 0 + for item in self._items(): + if type(item) is not dict or set(item) != {"text"} or type(item["text"]) is not str: + raise ResultContractError("result_value_invalid") + data = item["text"].encode("utf-8") + content_digest.update(data) + size += len(data) + characters += len(item["text"]) + yield item["text"] + if characters != self.character_count: + raise ResultContractError("result_content_integrity") + self._verify(content_digest, size, 1) + + def iter_value_bytes(self): + if self.result_kind not in {"structured-v1", "comparison-v1"}: + raise ResultContractError("result_kind_incompatible") + content_digest, size = hashlib.sha256(), 0 + comparison = bytearray() if self.result_kind == "comparison-v1" else None + for item in self._items(): + if ( + type(item) is not dict or set(item) != {"data"} or type(item["data"]) is not str + or not item["data"].isascii() + ): + raise ResultContractError("result_value_invalid") + data = item["data"].encode("ascii") + content_digest.update(data) + size += len(data) + if comparison is not None: + if size > MAX_VALUE_BYTES: + raise ResultContractError("result_comparison_too_large") + comparison.extend(data) + yield data + if comparison is not None: + _validate_value(self.result_kind, json.loads(comparison), self.completeness, self._source_snapshots) + self._verify(content_digest, size, self.reference.item_count) + + def _materialization_limit(self, max_bytes): + integer(max_bytes, minimum=1) + if max_bytes > MAX_VALUE_BYTES or self.reference.size_bytes > max_bytes: + raise ResultContractError("result_requires_streaming") + + def read_text(self, *, max_bytes=MAX_VALUE_BYTES): + self._materialization_limit(max_bytes) + return "".join(self.iter_text()) + + def read_value(self, *, max_bytes=MAX_VALUE_BYTES): + self._materialization_limit(max_bytes) + value = json.loads(b"".join(self.iter_value_bytes())) + _validate_value(self.result_kind, value, self.completeness, self._source_snapshots) + return value + + def preview(self, *, max_items=3, max_bytes=4096): + """Inspection only: never a full-reader replacement or an integrity receipt.""" + integer(max_items, minimum=1) + integer(max_bytes, minimum=1) + if max_items > MAX_PREVIEW_ITEMS or max_bytes > MAX_PREVIEW_BYTES: + raise ResultContractError("result_preview_limit") + if self.result_kind in _COLLECTION_KINDS and max_bytes < 2: + raise ResultContractError("result_preview_limit") + self.recheck() + if self.result_kind in _COLLECTION_KINDS: + rows, total = read_result_records( + self._collection(), self.reference.output_name, self._load_section, limit=max_items, + ) + value = [] + for row in rows: + _validate_item(self.result_kind, row, self.columns, self._source_snapshots) + if len(canonical_bytes([*value, row])) > max_bytes: + break + value.append(row) + truncated = len(value) != total + else: + fragments = self.iter_text() if self.result_kind in _TEXT_KINDS else ( + chunk.decode("ascii") for chunk in self.iter_value_bytes() + ) + parts, size, truncated = [], 0, False + for fragment in fragments: + encoded = fragment.encode("utf-8") + if size + len(encoded) > max_bytes: + parts.append(encoded[:max_bytes - size].decode("utf-8", errors="ignore")) + truncated = True + break + parts.append(fragment) + size += len(encoded) + value = "".join(parts) + self.recheck() + return { + "kind": self.result_kind, "value": value, "preview": True, + "truncated": truncated, "integrity_verified": False, + "item_count": self.reference.item_count, + } + + +class SavedAnalysisRecordSource: + """Explicit public-values projection of an already authorized SavedAnalysisInput. + + Owners call load_orchestration_analysis_input(..., bounded=True) and inject its + reader. The native manifest, full records, evidence, coverage and source + snapshots remain intact; no arbitrary content becomes analyze-final-v1. + """ + + kind = "records" + + def __init__(self, reader, *, columns): + validate_columns(columns, "records-v1") + manifest = reader.manifest + output = (manifest.get("outputs") or {}).get(manifest.get("authoritative_output")) or {} + if ( + manifest.get("contract_version") != "analyze-final-v1" or output.get("kind") != "records" + or (manifest.get("validation") or {}).get("status") != "valid" + or (manifest.get("execution") or {}).get("status") != "succeeded" + ): + raise ResultContractError("result_native_analysis_incomplete") + self.record_count = integer(manifest.get("record_count")) + self.columns = columns + self._reader = reader + self.recheck() + + def recheck(self): + self._reader.recheck() + + def metadata(self): + self.recheck() + return { + **self._reader.metadata(), + "coverage": deepcopy(self._reader.manifest.get("coverage") or {}), + "sources": deepcopy((self._reader.manifest.get("analysis_access") or {}).get("sources") or []), + } + + def iter_units(self): + self.recheck() + count = 0 + for unit in self._reader.iter_units(): + record = unit.get("record") + if type(record) is not dict: + raise ResultContractError("result_native_analysis_invalid") + validate_record(record.get("values"), self.columns) + count += 1 + if count > self.record_count: + raise ResultContractError("result_count_invalid") + yield deepcopy(unit) + if count != self.record_count: + raise ResultContractError("result_count_invalid") + self.recheck() + + def iter_records(self): + for unit in self.iter_units(): + values = unit["record"]["values"] + yield {column.name: values[column.name] for column in self.columns} diff --git a/application/single_app/functions_workflow_result_store.py b/application/single_app/functions_workflow_result_store.py index 7436d6507..6654221ad 100644 --- a/application/single_app/functions_workflow_result_store.py +++ b/application/single_app/functions_workflow_result_store.py @@ -24,6 +24,10 @@ 'orchestration_analysis_result_chunk' there with their real run_id and step_id. None of these payloads enter the message feed or orchestration UI step list. +Generic orchestration results reuse that same private run/step namespace and +mandatory lifecycle fence. Digest-keyed commits resolve their transport references +server-side; they do not claim the strict analyze-final-v1 result contract. + References contain only storage, schema_version (the storage format, not the envelope contract), sha256, size_bytes, and chunk_count. Paths are reconstructed from hashed identities. JSON is canonical ASCII, so page offsets count serialized @@ -51,6 +55,7 @@ CHAT_RESULT_RECORD_TYPE = "chat_analysis_result_chunk" ORCHESTRATION_RESULT_RECORD_TYPE = "orchestration_analysis_result_chunk" ANALYSIS_CONTROL_RECORD_TYPE = "analysis_work_unit_checkpoint" +ORCHESTRATION_RESULT_COMMIT_KEY = "orchestration-task-result-v1" RESULT_MEDIA_TYPE = "application/json" DEFAULT_MAX_RESULT_SIZE_MB = 500 MAX_COSMOS_CHUNK_BYTES = 256 * 1024 @@ -451,6 +456,42 @@ def save_orchestration( guard_token=guard_token, require_analysis_guard=require_analysis_guard, ) + def prepare_orchestration_result(self, user_id, conversation_id, run_id, step_id, *, guard_token): + """Register an authorized producer using the existing execution/deletion fence.""" + return self.prepare_analysis_attempt( + _orchestration_identity(user_id, conversation_id, run_id, step_id), token=guard_token, + ) + + def commit_orchestration_result( + self, user_id, conversation_id, run_id, step_id, reference, *, guard_token, + ): + """Commit a private digest-addressed manifest pointer, never a downloadable file.""" + identity = _orchestration_identity(user_id, conversation_id, run_id, step_id) + reference, manifest = self._read_manifest(identity, reference) + if manifest.get("analysis_fenced") is not True: + raise WorkflowResultIntegrityError("An orchestration result requires its lifecycle fence.") + key = f"{ORCHESTRATION_RESULT_COMMIT_KEY}:{reference['sha256']}" + row = { + **identity, "id": _analysis_control_id(identity, "final", key), + "type": ANALYSIS_CONTROL_RECORD_TYPE, "item_type": ANALYSIS_CONTROL_RECORD_TYPE, + "record_kind": "final", "key": key, "reference": reference, + } + self._write_analysis_record(identity, row, token=guard_token, immutable=True) + + def load_committed_orchestration_result(self, user_id, conversation_id, run_id, step_id, manifest_sha256): + """Resolve storage only from a producer-bound immutable server commit.""" + if not isinstance(manifest_sha256, str) or not _DIGEST_PATTERN.fullmatch(manifest_sha256): + raise WorkflowResultIntegrityError("The orchestration result digest is invalid.") + identity = _orchestration_identity(user_id, conversation_id, run_id, step_id) + key = f"{ORCHESTRATION_RESULT_COMMIT_KEY}:{manifest_sha256}" + row = self.read_analysis_checkpoint(identity, "final", key) + if row is None: + raise WorkflowResultIntegrityError("The orchestration result has not been committed.") + reference = _validate_reference(row.get("reference")) + if reference["sha256"] != manifest_sha256: + raise WorkflowResultIntegrityError("The orchestration result commit does not match.") + return self._load(identity, reference) + def _save(self, identity, result, *, guard_token=None, require_analysis_guard=False): """Persist using a validated result binding, with identical I/O.""" payload = self._serialize(result) diff --git a/docs/explanation/features/ANALYZE_RESULTS.md b/docs/explanation/features/ANALYZE_RESULTS.md index 56ae09ecf..d248a6394 100644 --- a/docs/explanation/features/ANALYZE_RESULTS.md +++ b/docs/explanation/features/ANALYZE_RESULTS.md @@ -1,5 +1,7 @@ # Saved Analyze results +**Version: 0.261.125** + Implemented in version: **0.261.109**, recorded in `application/single_app/config.py`. @@ -42,6 +44,14 @@ Storage version, result version, and calculation-specification version are independent. They are internal compatibility boundaries, not settings users must choose before asking a question. +The [orchestration result foundation](ORCHESTRATION_RENDERING_HARNESS.md), +implemented in **0.261.125** (Refs #1509), can wrap an already authorized bounded +`SavedAnalysisInput` in `SavedAnalysisRecordSource`. Its ordered public-values +projection retains complete records, evidence, coverage, and source snapshots +through the original reader. Generic text, comparisons, and structured values +have separate versioned kinds; they do not weaken `analyze-final-v1` validation. +This internal compatibility API does not change standalone Analyze or workflows. + ## Final data and coverage Window outputs are candidates, not automatically final records. Each accepted diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md index e0f697fac..14cf73805 100644 --- a/docs/explanation/features/CHAT_ORCHESTRATION.md +++ b/docs/explanation/features/CHAT_ORCHESTRATION.md @@ -1,6 +1,6 @@ # Chat Orchestration -**Version: 0.261.119** (tracked in `application/single_app/config.py`) +**Version: 0.261.125** (tracked in `application/single_app/config.py`) **Implemented in version: 0.261.086** **Knowledge phase added in version: 0.261.089** @@ -33,6 +33,12 @@ its required companions. This is a V2 interface feature. The classic interface is unchanged. +Version **0.261.125** adds the internal +[Gather / Reason / Render result foundation](ORCHESTRATION_RENDERING_HARNESS.md) +(Refs #1509). Its purpose labels, typed named outputs, and authorized retained +readers are foundation APIs only. They do not change the v1 phases/executor +described here, activate rendering, add settings, or publish files. + ## Dependencies - `enable_chat_orchestration` must be on. diff --git a/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md b/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md index b03b6ac19..8a5ec8d99 100644 --- a/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md +++ b/docs/explanation/features/ORCHESTRATION_CHECKPOINT_RECOVERY.md @@ -1,6 +1,6 @@ # Orchestration Checkpoint Recovery -**Version: 0.261.105** +**Version: 0.261.125** Implemented in version: **0.261.105**, recorded in `application/single_app/config.py`. @@ -105,6 +105,13 @@ completed work remains bound to its saved checkpoint in an earlier attempt. ## Durable state and validation +The [retained-result foundation](ORCHESTRATION_RENDERING_HARNESS.md), implemented +in **0.261.125** (Refs #1509), stores large original datasets in private result +sections and exposes small immutable descriptors. It does not add production +`RunContext` or checkpoint fields in this layer. Existing v1 state serialization, +input fingerprints, source-bound Analyze references, and recovery behavior stay +unchanged; a foundation descriptor is not automatically a reusable checkpoint. + Completed results include their evidence, notes, citations, artifact references, required context changes, and execution provenance. A summary alone cannot reconstruct these values. diff --git a/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md b/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md new file mode 100644 index 000000000..13ac35f87 --- /dev/null +++ b/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md @@ -0,0 +1,455 @@ +# Gather / Reason / Render result foundation + +**Version: 0.261.125** + +Implemented in version: **0.261.125**, recorded in +`application/single_app/config.py`. + +Refs [#1509](https://github.com/microsoft/simplechat/issues/1509). +This implements the **M0 contract baseline and M1 retained-result facade only**, +not the complete orchestration/rendering epic. + +## Purpose and availability + +The foundation gives future orchestration adapters a common way to retain exact +results and bind a later consumer to a named output. A narrative finding, a +comparison, a table, and prepared source-free content do not have to masquerade +as a native Analyze result or be reconstructed from a preview. + +Gather, Reason, and Render are purpose labels, not a required stage order. +Knowledge collection belongs to Gather. Analyze, Compare, tabular analysis, +synthesis, and drafting belong to Reason. Render will turn prepared results into +messages or files without undisclosed source retrieval or model work. The new +task contract rejects a Gather/Render label for the existing +`document_analyze`, `document_compare`, and `tabular_analyze` capabilities. + +There is **no user-facing enablement or new setting** in this layer. Existing +v1 plans, capability phases, executor behavior, `RunContext`, checkpoint fields, +standalone tools, workflows, and download routes are unchanged. The following +APIs are internal foundation interfaces; production adapters and the scheduler +do not automatically call them. + +New result writes are private retained data. They do not create chat artifacts, +download links, workspace documents, or published files. Existing native tool +artifact behavior remains compatible; that is not an implementation of the +future orchestration file-publication policy. + +## Dependencies and ownership + +| Component | Responsibility | +| --- | --- | +| `functions_orchestration_result_contracts.py` | Standard-library-only frozen descriptors, completeness, ordered schemas, named bindings, and dependency validation. | +| `functions_orchestration_results.py` | Injected producer/source authorization, private result persistence, complete readers, bounded previews, and explicit saved-Analyze projection. | +| `functions_workflow_result_store.py` | Existing immutable Cosmos/Blob transport and cancellation/deletion fences, plus three generic orchestration commit methods. | +| `functions_workflow_collections.py` | Bounded record-tree leaves/indexes and one aggregate write budget. | +| `functions_workflow_results.py` | Existing bounded section readers, including collection previews. | +| `functions_saved_analysis.py` | Existing native Analyze owner and authorized `SavedAnalysisInput`; no weakened `analyze-final-v1` validation. | +| `functions_analysis_access.py` and `content_screening.access` | Current source authorization, revision checks, and screening holds. | + +Runtime owners supply initialized storage and access readers. The new modules do +not import `config`, routes, settings owners, saved-analysis owners, renderers, +artifact publishers, or model clients. They do not create another Cosmos/Blob +client, fabricate a workflow, or introduce global memory. + +## Versioned contracts + +| Contract | Wire version | +| --- | --- | +| `TaskResult` | `orchestration-task-result-v1` | +| `ResultRef` | `orchestration-result-ref-v1` | +| `InputBinding` | `orchestration-input-binding-v1` | +| `Completeness` | `orchestration-completeness-v1` | + +The public frozen descriptors are: + +```python +ProducerIdentity( + user_id, conversation_id, run_id, attempt_index, + step_id, capability_id, contract_version, +) +Coverage(expected, completed, unit) +Completeness( + status, expected_count, actual_count, coverage, + validation, checks, limitations, preview=False, +) +RecordColumn(name, value_type, nullable=False) +ResultRef( + producer, output_name, kind, manifest_sha256, content_sha256, + size_bytes, item_count, completeness, columns=(), character_count=None, +) +TaskResult(producer, role, status, outputs) +InputBinding(step_id=None, output_name=None, existing_result=None) +``` + +Descriptors use `to_dict()` and strict `from_dict()` methods. Tuples serialize as +JSON arrays; unknown fields, missing fields, unsupported versions, coercible but +incorrect types, and duplicate names are errors. An `InputBinding` dictionary +contains all three alternative fields, with unused fields explicitly `null`. +`task.output(name)` selects one exact output; a missing name is not an empty +result. + +`ResultRef` contains no backend name, blob path, storage handle, client, +credential, callback, prompt, or retained dataset. The server resolves its +manifest digest through a producer-bound committed pointer. The manifest digest +binds all named outputs and their lineage. Each content digest covers canonical +compact JSON for collections/structured values, or original UTF-8 bytes for +text. These are data-integrity digests, not future exported-file digests. + +### Honest completeness + +The states are `pending`, `partial`, `complete`, `invalid`, `cancelled`, `failed`, +and `unavailable`. Completion is never inferred from nonempty prose or a +successful storage operation. + +`complete` requires `validation="valid"`, explicit nonempty checks, equal known +expected/actual item counts, fully known completed coverage, and `preview=False`. +Coverage units are `sources`, `work_units`, `records`, or `items`. A legitimate +empty collection can therefore be complete; an unavailable collection cannot +be substituted with an empty one. + +`partial` requires valid/partial validation and explicit limitations. It is not +readable unless the caller deliberately supplies `allow_partial=True`. +Pending/invalid/failed/cancelled/unavailable outputs remain unreadable even with +that option. Accepting a partial upstream result cannot promote a child output +to complete. + +Counts, schemas, persisted content, and final-item presence are checked by the +facade. Producer checks and coverage describe work the producer actually +performed; they are not an independent factual review or proof that every +possible finding was identified. + +### Implemented result kinds + +| Kind | Retained value and complete reader | +| --- | --- | +| `records-v1` | JSON objects with an explicit ordered schema; `iter_records()`. | +| `text-v1` | Exact text; `iter_text()` or bounded `read_text()`. | +| `markdown-v1` | Exact prepared Markdown; the same text readers, with a distinct kind. | +| `structured-v1` | Strict JSON values, including nested objects/arrays, scalars and null; `iter_value_bytes()` or bounded `read_value()`. | +| `source-set-v1` | Exact authorized source snapshots; `iter_items()`. | +| `evidence-set-v1` | Objects containing exactly `evidence_id`, `source`, and `text`; `iter_items()`. | +| `comparison-v1` | Explicit completed/failed comparisons and target identities; `iter_value_bytes()` or `read_value()`. | + +Records must contain exactly the declared columns. Supported column types are +`string`, `integer`, `number`, `boolean`, `object`, `array`, and `json`, with null +accepted only when `nullable=True`. Boolean values are not integers. Readers +restore declared column order and preserve leading zeros, formula-like strings, +Unicode, nested values, nulls, and booleans without flattening or coercion. + +Source and evidence entries must match the full retained lineage snapshot, +including revision/version and any recorded content digest, not just a document +ID. Source identities and evidence IDs must not repeat within their collections. + +The `comparison-v1` value has exactly this shape: + +```json +{ + "left_document_id": "left-id", + "right_document_ids": ["right-id"], + "items": [ + { + "right_document_id": "right-id", + "right_document_name": "Comparison source", + "text": "Prepared comparison findings." + } + ], + "failed_document_ids": [] +} +``` + +Targets must be unique, authorized in the lineage, and accounted for exactly +once as completed or failed. `expected_count` counts requested right-hand +targets; `actual_count` counts completed comparisons. Failure prose is not a +completed comparison. A complete result cannot contain failed targets. + +Prepared report/slide document schemas are **not** supported result kinds yet. +Storing arbitrary JSON does not advertise a validated Office/PDF layout contract. + +## Injected access and storage interfaces + +```python +OrchestrationResultAccess( + *, + user_id, + conversation_id, + read_conversation, + read_run, + source_resolver=None, + source_metadata_reader=None, +) +OrchestrationResults(store, access, *, max_result_bytes=None) +``` + +The callbacks are server-owned: + +| Callback | Required behavior | +| --- | --- | +| `read_conversation(conversation_id)` | Return the current conversation record, including its true owner and deletion state. | +| `read_run(run_id)` | Return the current run record, including owner, conversation, attempt, current plan steps, cancellation and deletion state. | +| `source_resolver(document_ids, **scope)` | Implement the existing `resolve_authorized_source_manifest` protocol; current access/scope/version/revision information must come from authorized source metadata, not the request. | +| `source_metadata_reader(document_id, user_id, group_id=None, public_workspace_id=None)` | Enforce the actor's current source/workspace access and return current screening metadata using the existing screening reader protocol. | + +Source-free results need no source callbacks. Grounded results fail closed if a +required callback or source is unavailable. There is no application-settings or +client-factory fallback. + +The facade methods are: + +```python +service.persist_task_result( + *, + producer, + role, + status, + outputs, + sources, + origin, + guard_token, + upstream=(), + source_policy="current", + allow_partial_inputs=False, +) # -> TaskResult + +service.open_result( + reference, *, allow_partial=False, require_current_sources=False, +) # -> OrchestrationResultReader + +service.resolve_input( + spec, *, consumer, task_results, existing_results=None, +) # -> OrchestrationResultReader +``` + +Each output is `NamedOutput(name, kind, value, completeness, columns=())`. +Collection values may be one-pass iterators. The guard token must be the +server-owned token for that actual producer attempt; it is not a model argument +or part of a serialized result reference. + +The shared `WorkflowResultStore` adds only: + +```python +store.prepare_orchestration_result( + user_id, conversation_id, run_id, step_id, *, guard_token, +) +store.commit_orchestration_result( + user_id, conversation_id, run_id, step_id, reference, *, guard_token, +) +store.load_committed_orchestration_result( + user_id, conversation_id, run_id, step_id, manifest_sha256, +) +``` + +Private sections reuse the existing orchestration result namespace, including +its historical `orchestration_analysis_result_chunk` name. This is a transport +name, not a claim that generic content satisfies Analyze's native schema. +Only the final immutable commit makes a descriptor readable. A caller-provided +digest for an uncommitted section cannot bypass that boundary. + +### Access, lineage, and attempt boundaries + +Reuse is original-owner and same-conversation only. Every read checks the +conversation and run owners, deletion state, exact attempt, and one matching +enabled producer step/capability. Historical v1 runs without `attempt_index` +retain their established original-attempt value of 1. Writes also require a +running run with no cancellation request or successor attempt. + +`sources` contains exact `analysis_source_snapshot` dictionaries: +`document_id`, `scope`, `scope_id`, `source_version`, `source_revision`, and +optional `content_sha256`. Cached authorization decisions and storage locators +are not accepted as source snapshots. + +`origin="generated"` requires no source/upstream lineage. `origin="grounded"` +requires actual sources or upstream results. Upstream references are recursively +reauthorized, cycle-checked, and bounded. Their source snapshots remain part of +the child's authorization boundary. + +The default `source_policy="current"` rejects changed source versions, +revisions, and recorded content digests. Explicit `source_policy="snapshot"` +allows the historical retained content while reporting +`source_snapshot_changed=True`. It never skips current source access or screening. +`require_current_sources=True` can impose current-snapshot validation on a reader, +including its upstream lineage. + +Opening a reader is not a permanent access grant. Authorization is repeated when +loading sections and after full reads. Deletion preserves existing lifecycle +tombstones; a late or differently tokened worker cannot recreate deleted results. +Cancellation fences new writes without deleting already completed readable +results. No fence is replaced by a permissive fallback. + +## Named bindings without a runtime rewrite + +```python +InputSpec(name, binding, kinds, allow_partial=False) +OutputSpec(name, kind) +StepBindings(step_id, enabled, outputs=(), inputs=(), depends_on=()) +validate_input_bindings(steps, *, existing_results=None, max_steps=64) +``` + +The server supplies these specifications and accepted kinds. A model binding can +select a producing step plus output name, or a named alias in an explicitly +admitted existing-result catalog. It cannot supply storage references, actor +identity, projections, or arbitrary callbacks. + +Validation returns dependency tuples without mutating, sorting, repairing, +trimming, or executing a production plan. Missing/disabled producers, unknown +outputs, incompatible kinds, duplicate names, cycles, and step-budget overflow +are errors. It does not make a requested output disappear to fit a limit. + +At consumption, a step/output binding must match the consumer's actual run and +attempt. Reuse across attempts requires an explicit server-admitted existing +result alias; it is not silently rebound to a similarly named step. Alias +resolution still rechecks the original producer and every source. An alias +cannot introduce a direct self-binding. + +## Using retained results from an owning adapter + +The following internal pattern assumes the producer has completed and validated +the prepared value, and that the owner has already supplied its authorized +producer identity, access callbacks, store, and lifecycle token: + +```python +from functions_orchestration_result_contracts import Completeness, Coverage, TaskResult +from functions_orchestration_results import NamedOutput, OrchestrationResults + +service = OrchestrationResults(store, access) +task = service.persist_task_result( + producer=producer, + role="reason", + status="complete", + outputs=[ + NamedOutput( + "prepared", + "structured-v1", + {"title": "Prepared content", "sections": []}, + Completeness( + "complete", 1, 1, Coverage(1, 1, "work_units"), "valid", + ("prepared_value_schema",), (), + ), + ), + ], + sources=[], + origin="generated", + guard_token=guard_token, +) + +# The owning layer retains only this small descriptor, not the dataset. +descriptor = task.to_dict() +restored = TaskResult.from_dict(descriptor) +reader = service.open_result(restored.output("prepared")) +prepared_value = reader.read_value() +``` + +This example does not add a field to existing production checkpoints. Wiring +descriptor retention into new runtime adapters is later work. Reconstructing an +`OrchestrationResults` instance with initialized handles and the same persisted +descriptor reads the original retained data, rather than rerunning its producer. + +### Reader protocol and integrity + +Records readers implement the existing export-source protocol: +`kind == "records"`, `record_count`, `columns`, `iter_records()`, and `recheck()`. +`columns` is a tuple of `RecordColumn` objects, not an array of inferred names. +Other reader `kind` values remain their explicit versioned result kinds. +`result_kind` always retains the versioned kind. + +Text/Markdown readers expose `character_count` as the verified Unicode character +count, distinct from UTF-8 byte size. Structured/comparison readers expose +`read_value()` and streamed canonical JSON through `iter_value_bytes()`. +`metadata()` returns the reference, distinct source count, origin, source policy, +and source-change flag. + +**Consumers must exhaust a full iterator before accepting or publishing its +result.** Final count/digest checks and the last access recheck run at exhaustion. +A preview or a partially consumed iterator is not an integrity receipt. +Exhausting an explicitly allowed partial result verifies that retained subset, +not complete original coverage. Future export adapters must preserve this +distinction rather than infer readiness from a record count. + +`preview(max_items=3, max_bytes=4096)` returns an explicitly marked preview with +`integrity_verified=False`. Collection previews contain bounded records; +text/Markdown and structured/comparison previews contain text fragments. A JSON +preview need not be a parseable complete JSON value. + +### Bounds and performance + +| Bound | Value | +| --- | --- | +| Outputs per task / inputs per step | 32 | +| Ordered record columns | 256 | +| Binding steps | At most 64; an owning layer can pass a smaller budget. | +| Serialized descriptor / final manifest / lineage | 128 KiB each. | +| Collection leaf budget | Existing 100-record / 128-KiB pages and bounded indexes. | +| Text / JSON fragment size | 4,096 Unicode characters / 16,384 ASCII characters. | +| Materialized text or structured value | At most 8 MiB; larger results require streaming. | +| Comparison value | At most 8 MiB, including semantic count validation during streamed reads. | +| Preview | At most 20 items / 16 KiB. | +| Upstream references / traversed lineage results | 64 / 256. | +| Aggregate retained bytes | `max_result_bytes`, defaulting to the injected store's `max_size_bytes`; sections and indexes share one budget. | + +Oversized records, manifests, or aggregate data fail rather than being shortened. +Large datasets live in bounded private sections, not an 8-MiB orchestration +checkpoint. Reauthorization has real metadata-I/O cost, especially with many +sources or upstream results; offline scale fixtures do not claim cloud throughput. + +## Existing saved Analyze and JSON compatibility + +An owner can pass the real bounded reader returned by +`load_orchestration_analysis_input(..., bounded=True)` to: + +```python +SavedAnalysisRecordSource(reader, *, columns) +``` + +This adapter requires a succeeded, valid `analyze-final-v1` records result. +`iter_records()` explicitly projects each original record's `values` through the +supplied ordered schema. `iter_units()` preserves full original record identities +and evidence; `metadata()` retains native coverage and source snapshots. All +reads delegate to the existing authorized native reader. + +This is a separate compatibility interface, not a conversion of arbitrary +prose or Compare output into Analyze. It does not modify native persistence, +standalone generated artifacts, or workflow behavior. + +The existing `exact_records_v1` JSON export accepts a complete generic records +reader without any new export profile. Other formats remain outside this PR; +in particular, passing CSV to that unchanged profile still fails. Neither the +facade nor the compatibility adapter invokes an export or publishes a file. + +See [Saved Analyze results](ANALYZE_RESULTS.md), +[Chat orchestration](CHAT_ORCHESTRATION.md), and +[Checkpoint recovery](ORCHESTRATION_CHECKPOINT_RECOVERY.md). + +## Verification and remaining scope + +The new regression suites are: + +```powershell +python -m pytest .\functional_tests\test_orchestration_result_contracts.py .\functional_tests\test_orchestration_results.py .\functional_tests\test_orchestration_result_imports.py -q +``` + +They exercise real production contracts/transport with external I/O doubles: +narrative Analyze and native foreground/durable fixtures; pending versus complete +data; source-free and multiple outputs; strict schemas/bindings; first/last-record +reuse by two readers after persistence/restart; a 30,000-row dataset exceeding +8 MiB; large text/structured data; count/digest/final-record corruption; current +source/producer/screening checks; explicit historical snapshots; stale tokens; +cancel/delete races; no artifact publication; unchanged v1 checkpoint +fingerprints; and the existing JSON export protocol. + +Cold-import checks use fresh normal and optimized interpreters with network +blocked. They cover both lower-level import orders, required storage operations +and failure paths, and normal web and separate scheduler imports using real +runtime modules rather than a fake `config`. + +Focused existing regressions cover Analyze storage/source access, orchestration +saved-result integration, checkpoint recovery/access, generated-file saved-record +exports, and the standalone v1 plan-schema runner. Legacy boolean-return scripts +must use their script runner; a pytest invocation alone is not evidence that +their returned result was successful. + +There is no M2+ adapter rollout, new graph scheduler, semantic compilation, +message/file rendering, render-task UI, automatic file retry policy, or +per-file publication in this foundation. CSV/XLSX/DOCX/PPTX/PDF/JSON/XML/YAML/ +Markdown/TXT orchestration output activation, prepared report/slide schemas, +independent file-task outcomes, and initial-plus-two automatic render attempts +follow in later reviewable layers. Workflows, media generation, cross-conversation +memory, and external delivery are outside this harness scope. diff --git a/functional_tests/test_orchestration_result_contracts.py b/functional_tests/test_orchestration_result_contracts.py new file mode 100644 index 000000000..008c56306 --- /dev/null +++ b/functional_tests/test_orchestration_result_contracts.py @@ -0,0 +1,313 @@ +# test_orchestration_result_contracts.py +""" +Executable M0 result/binding and compatibility contracts, not a harness rollout. +Version: 0.261.125 +Implemented in: 0.261.125 + +These tests exercise strict new APIs alongside the unchanged v1 plan/checkpoint +and native Analyze contracts. They do not claim later runtime/render milestones. +""" + +from copy import deepcopy +from dataclasses import FrozenInstanceError, replace +import json +from types import SimpleNamespace + +import pytest + +from functions_orchestration_result_contracts import ( + Completeness, + Coverage, + InputBinding, + InputSpec, + OutputSpec, + ProducerIdentity, + RecordColumn, + ResultContractError, + ResultRef, + StepBindings, + TaskResult, + canonical_digest, + validate_input_bindings, + validate_json, + validate_record, +) +from functions_orchestration_results import NamedOutput +from functions_native_analysis_results import adapt_native_analysis_result +from functions_workflow_results import build_orchestration_analysis_result +from test_support.orchestration_results import ( + COLUMNS, ROWS, SOURCE_FREE_VALUE, TABULAR_DURABLE, TABULAR_FOREGROUND, + ResultFixture, complete, native_analysis, +) +from test_support.versioning import assert_app_version_at_least + + +def reference(): + return ResultRef( + ProducerIdentity("owner", "conversation", "run", 1, "analyze", "document_analyze", "analyze-final-v1"), + "findings", "records-v1", "a" * 64, canonical_digest(ROWS), len(json.dumps(ROWS)), 3, complete(3), COLUMNS, + ) + + +def steps(): + return [ + StepBindings("prepare", True, (OutputSpec("findings", "records-v1"),)), + StepBindings("inspect", True, (OutputSpec("notes", "markdown-v1"),), ( + InputSpec("data", InputBinding("prepare", "findings"), ("records-v1",)), + )), + StepBindings("gather_again", True, (OutputSpec("sources", "source-set-v1"),), depends_on=("inspect",)), + ] + + +def test_version_tracks_foundation(): + assert_app_version_at_least("0.261.125") + + +def test_frozen_named_outputs_round_trip_and_no_transport_handles(): + saved = reference() + task = TaskResult(saved.producer, "reason", "complete", (saved,)) + encoded = task.to_dict() + restored = TaskResult.from_dict(json.loads(json.dumps(encoded))) + assert restored == task + with pytest.raises(FrozenInstanceError): + saved.kind = "structured-v1" + with pytest.raises(FrozenInstanceError): + saved.columns[0].name = "changed" + encoded["outputs"][0]["columns"][0]["name"] = "mutated" + assert saved.columns[0].name == "id" + serialized = json.dumps(task.to_dict()) + for forbidden in ("result_ref", "storage", "chunk_count", "blob", "token", "prompt", "callback"): + assert forbidden not in serialized + + +@pytest.mark.parametrize("capability", ["document_analyze", "document_compare", "tabular_analyze"]) +def test_analysis_capabilities_are_reason_labels_not_a_mandatory_stage(capability): + saved = reference() + producer = replace(saved.producer, capability_id=capability) + saved = replace(saved, producer=producer) + result = TaskResult(producer, "reason", "complete", (saved,)) + assert result.role == "reason" + with pytest.raises(ResultContractError): + replace(result, role="gather") + + +@pytest.mark.parametrize("changes", [ + {"status": "complete", "validation": "not_validated"}, + {"status": "complete", "checks": ()}, + {"expected_count": None}, + {"actual_count": 2}, + {"coverage": Coverage(None, 1, "sources")}, + {"coverage": Coverage(2, 1, "sources")}, + {"preview": True}, + {"actual_count": True}, + {"status": "successful"}, +]) +def test_completeness_cannot_invent_success(changes): + with pytest.raises(ResultContractError): + replace(complete(3), **changes) + + +@pytest.mark.parametrize("status", ["pending", "invalid", "unavailable", "failed", "cancelled"]) +def test_nonfinal_states_are_not_readable_even_with_partial_opt_in(status): + state = complete(0, status=status, expected=3) + with pytest.raises(ResultContractError): + state.require_readable(allow_partial=True) + + +def test_partial_requires_explicit_acceptance_and_limitations(): + state = complete(2, status="partial", expected=3) + with pytest.raises(ResultContractError): + state.require_readable() + state.require_readable(allow_partial=True) + with pytest.raises(ResultContractError): + replace(state, limitations=()) + with pytest.raises(ResultContractError): + replace(state, validation="not_validated") + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), {1: "coerced"}, (1, 2), object(), lambda: None]) +def test_json_rejects_runtime_objects_callbacks_and_lossy_types(value): + with pytest.raises(ResultContractError): + validate_json(value) + + +def test_ordered_schema_rejects_extra_missing_and_wrong_typed_fields(): + for mutation in ( + {**ROWS[0], "hidden_source_id": "not-public"}, + {key: value for key, value in ROWS[0].items() if key != "amount"}, + {**ROWS[0], "enabled": 1}, + {**ROWS[0], "amount": False}, + ): + with pytest.raises(ResultContractError): + validate_record(mutation, COLUMNS) + for record in ROWS: + validate_record(record, COLUMNS) + with pytest.raises(ResultContractError): + replace(reference(), columns=(RecordColumn("id", "string"), RecordColumn("id", "integer"))) + + +@pytest.mark.parametrize("injected", [ + {"storage": "blob"}, {"blob_path": "guessed"}, {"user_id": "foreign"}, + {"projection": {"flatten": True}}, {"result_ref": {"sha256": "a" * 64}}, +]) +def test_model_bindings_cannot_supply_identity_storage_or_projections(injected): + value = InputBinding("prepare", "findings").to_dict() + with pytest.raises(ResultContractError): + InputBinding.from_dict({**value, **injected}) + + +def test_binding_validation_infers_dependencies_without_stage_sort_or_mutation(): + planned = steps() + original = deepcopy(planned) + dependencies = validate_input_bindings(planned) + assert dependencies == {"prepare": (), "inspect": ("prepare",), "gather_again": ("inspect",)} + assert planned == original + assert list(dependencies) == ["prepare", "inspect", "gather_again"] + + +@pytest.mark.parametrize("case", [ + "missing", "disabled", "disabled_branch", "kind", "cycle", "self", "missing_output", "budget", "duplicate", +]) +def test_bad_dependencies_are_rejected_not_dropped(case): + planned = steps() + options = {} + if case == "missing": + planned = planned[1:] + elif case == "disabled": + planned[0] = replace(planned[0], enabled=False) + elif case == "disabled_branch": + planned = [replace(step, enabled=False) for step in planned] + elif case == "kind": + planned[0] = replace(planned[0], outputs=(OutputSpec("findings", "text-v1"),)) + elif case == "cycle": + planned[0] = replace(planned[0], depends_on=("gather_again",)) + elif case == "self": + planned[0] = replace(planned[0], depends_on=("prepare",)) + elif case == "missing_output": + planned[0] = replace(planned[0], outputs=()) + elif case == "budget": + options["max_steps"] = 2 + else: + planned.append(planned[0]) + original = deepcopy(planned) + with pytest.raises(ResultContractError): + validate_input_bindings(planned, **options) + assert planned == original + + +def test_existing_result_bindings_require_explicit_server_catalog(): + spec = InputSpec("data", InputBinding(existing_result="selected_findings"), ("records-v1",)) + planned = [StepBindings("inspect", True, inputs=(spec,))] + with pytest.raises(ResultContractError): + validate_input_bindings(planned) + with pytest.raises(ResultContractError): + validate_input_bindings(planned, existing_results={"selected_findings": reference().to_dict()}) + resolved = validate_input_bindings(planned, existing_results={"selected_findings": reference()}) + assert resolved == {"inspect": ()} + + +def test_narrative_analyze_and_tabular_fixtures_preserve_native_contract(): + native = native_analysis() + envelope = build_orchestration_analysis_result( + {"analysis_result": native}, user_id="owner", conversation_id="conversation-1", run_id="run-1", step_id="analyze", + ) + assert envelope["contract_version"] == "analyze-final-v1" + assert envelope["outputs"]["records"]["value"] == native["authoritative_result"]["value"] + assert envelope["presentation"]["artifacts"] == [] + assert envelope["record_count"] == 3 + assert TABULAR_FOREGROUND["value"] == ROWS + assert TABULAR_DURABLE == {"status": "running", "run_id": "native-job-1"} + pending = TaskResult(reference().producer, "reason", "pending", ()) + assert pending.outputs == () + with pytest.raises(ResultContractError): + replace(pending, status="complete") + + +def test_foreground_and_durable_native_api_fixtures_do_not_promote_a_preview(): + fixture = ResultFixture() + native_source = {**fixture.sources["document-1"], "source_kind": "tabular", "file_name": "inventory.csv"} + foreground = adapt_native_analysis_result( + user_id="owner", conversation_id="conversation-1", source=native_source, + complete_output={ + **deepcopy(TABULAR_FOREGROUND), "source_file_name": "inventory.csv", + "source_authorization": {"source": "workspace"}, + }, + source_resolver=fixture.resolve, + ) + values = [record["values"] for record in foreground["authoritative_result"]["value"]] + assert values == ROWS + assert foreground["generated_tabular_outputs"] == [] + assert foreground["analysis_validation"]["status"] == "valid" + calls = [] + native_state = deepcopy(TABULAR_DURABLE) + + def read_native(user_id, conversation_id, run_id, selected_source): + calls.append((user_id, conversation_id, run_id, selected_source["document_id"])) + return deepcopy(native_state) + + pending = adapt_native_analysis_result( + user_id="owner", conversation_id="conversation-1", source=native_source, + generated_outputs=[{"run_id": "native-job-1", "status": "running"}], + native_reader=read_native, source_resolver=fixture.resolve, + ) + assert pending["execution_status"] == "pending" + assert pending["analysis_validation"]["status"] == "pending" + assert pending["authoritative_result"]["value"] == [] + assert pending["native_result_references"] == [{"run_id": "native-job-1", "status": "running"}] + legacy_artifacts = [{"artifact_message_id": "existing-native-output", "output_format": "json"}] + native_state.update({ + "status": "completed", "kind": "records", "value": ROWS, "source_row_count": 3, + "batch_count": 1, "completed_batches": 1, "artifacts": legacy_artifacts, + }) + finished = adapt_native_analysis_result( + user_id="owner", conversation_id="conversation-1", source=native_source, + generated_outputs=[{"run_id": "native-job-1", "status": "completed"}], + native_reader=read_native, source_resolver=fixture.resolve, + ) + values = [record["values"] for record in finished["authoritative_result"]["value"]] + assert values == ROWS + assert finished["generated_tabular_outputs"] == legacy_artifacts + assert calls == [("owner", "conversation-1", "native-job-1", "document-1")] * 2 + + +def test_source_free_and_multiple_outputs_are_named_data_not_files(): + fixture = ResultFixture() + task = fixture.save(grounded=False, outputs=[ + NamedOutput("configuration", "structured-v1", SOURCE_FREE_VALUE, complete(1)), + NamedOutput("report", "markdown-v1", "# Prepared content\n\nNo new source work.", complete(1)), + NamedOutput("findings", "records-v1", ROWS, complete(3), COLUMNS), + ]) + restored = TaskResult.from_dict(task.to_dict()) + restarted = fixture.restart() + value = restarted.open_result(restored.output("configuration")).read_value() + text = restarted.open_result(restored.output("report")).read_text() + assert value == SOURCE_FREE_VALUE + assert text.endswith("No new source work.") + assert tuple(result.output_name for result in restored.outputs) == ("configuration", "report", "findings") + assert restored.role == "reason" + assert all("artifact" not in record for record in fixture.container.items.values()) + + +def test_v1_checkpoint_state_and_fingerprint_ignore_unused_foundation(): + from functions_orchestration_checkpoints import context_binding, context_state, restore_context, step_input_fingerprint + + context = SimpleNamespace(user_message="Review retained findings", memory_context={}, selected_document_ids=[]) + plan = {"steps": [{ + "step_id": "a", "capability_id": "document_analyze", "arguments": {"document_ids": ["d1"]}, + "depends_on": [], "enabled": True, "optional": False, + }]} + binding = context_binding(context, plan, {}) + state = context_state(context) + fingerprint = step_input_fingerprint(plan["steps"][0], context, binding) + context.saved_analyses = [] + context.analysis_result_contexts = [] + context.unused_foundation_result = reference().to_dict() + unchanged_binding = context_binding(context, plan, {}) + unchanged_state = context_state(context) + unchanged_fingerprint = step_input_fingerprint(plan["steps"][0], context, binding) + restore_context(context, {"state": state}) + restored_state = context_state(context) + assert unchanged_binding == binding + assert unchanged_state == restored_state == state + assert unchanged_fingerprint == fingerprint + assert "unused_foundation_result" not in restored_state diff --git a/functional_tests/test_orchestration_result_imports.py b/functional_tests/test_orchestration_result_imports.py new file mode 100644 index 000000000..99eacd8d5 --- /dev/null +++ b/functional_tests/test_orchestration_result_imports.py @@ -0,0 +1,161 @@ +# test_orchestration_result_imports.py +""" +Real-module result-store cold imports and web/scheduler compatibility. +Version: 0.261.125 +Implemented in: 0.261.125 + +Fresh normal/optimized interpreters block network access. Only external I/O is +doubled; config, settings, app, scheduler and result modules are never replaced. +""" + +import ast +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "application" / "single_app" +TESTS = ROOT / "functional_tests" +EARLY_PROBE = r''' +import builtins +import importlib +import socket +import sys +from unittest.mock import patch +from azure.core.exceptions import AzureError + +sys.path[:0] = sys.argv[1:3] +real_import = builtins.__import__ +forbidden = { + "config", "functions_settings", "functions_saved_analysis", + "functions_generated_file_exports", "functions_artifact_publication", + "route_backend_orchestration", "route_backend_chats", +} +network_attempts = [] + +def no_network(*args, **kwargs): + network_attempts.append(True) + raise AssertionError("Unexpected external I/O") + +def guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + if level == 0 and name in forbidden: + raise AssertionError("Unexpected result owner import: " + name) + return real_import(name, globals, locals, fromlist, level) + +def check(condition, message): + if not condition: + raise AssertionError(message) + +with patch.object(socket.socket, "connect", no_network), patch.object(builtins, "__import__", guarded_import): + for name in sys.argv[3:]: + importlib.import_module(name) + from test_support.orchestration_results import ResultFixture, ROWS + from functions_orchestration_result_contracts import TaskResult + from functions_orchestration_results import OrchestrationResultAccess, ResultContractError + fixture = ResultFixture(blob=True) + saved = fixture.save(grounded=False) + checkpoint = saved.to_dict() + restored = TaskResult.from_dict(checkpoint) + reader = fixture.restart().open_result(restored.output("findings")) + records = list(reader.iter_records()) + check(records == ROWS, "Complete result changed during cold restart") + check(records[-1]["id"] == "last", "Final record was not read") + check(bool(fixture.blobs.uploads), "Required persistence disappeared under optimized Python") + try: + OrchestrationResultAccess( + user_id="owner", conversation_id="conversation-1", + read_conversation=None, read_run=None, + ) + except ResultContractError: + pass + else: + raise AssertionError("Uninitialized access silently bootstrapped") + fixture.container.fail_writes = True + try: + fixture.save(grounded=False) + except AzureError: + pass + else: + raise AssertionError("Storage failure became a successful result") + check(not network_attempts, "A blocked network failure was swallowed") + check(not forbidden.intersection(sys.modules), "Lower-level result access imported a runtime owner") +print("PASS: exact retained results, failure paths and owner-free cold imports") +''' + +APP_PROBE = r''' +import importlib +import sys + +sys.path[:0] = sys.argv[1:3] +from test_support.offline_bootstrap import offline_app_imports + +with offline_app_imports() as environment: + for name in sys.argv[3:]: + importlib.import_module(name) + contracts = importlib.import_module("functions_orchestration_result_contracts") + results = importlib.import_module("functions_orchestration_results") + store = importlib.import_module("functions_workflow_result_store") + scheduler = importlib.import_module("background_tasks") + if results.ResultRef is not contracts.ResultRef: + raise AssertionError("Bootstrap replaced the result contract") + if not callable(store.WorkflowResultStore.load_committed_orchestration_result): + raise AssertionError("Shared store lost the generic result reader") + if "app" in sys.argv[3:] and not hasattr(importlib.import_module("app"), "app"): + raise AssertionError("Normal web bootstrap did not complete") + if not callable(scheduler.check_m365_workflow_continuations_once): + raise AssertionError("Normal scheduler bootstrap did not complete") + if environment.network_attempts: + raise AssertionError("Cold import swallowed a network attempt") +print("PASS: real web and scheduler imports with result foundation") +''' + + +def run_probe(probe, order, optimized): + command = [sys.executable, "-B"] + if optimized: + command.append("-O") + process = subprocess.run( + command + ["-c", probe, str(APP), str(TESTS), *order], + capture_output=True, text=True, timeout=180, check=False, + ) + assert process.returncode == 0, process.stdout[-8000:] + process.stderr[-8000:] + assert "PASS:" in process.stdout + + +@pytest.mark.parametrize("optimized", [False, True]) +@pytest.mark.parametrize("order", [ + ("functions_orchestration_results", "functions_workflow_result_store"), + ("functions_workflow_result_store", "functions_orchestration_results"), +]) +def test_early_real_modules_and_injected_storage_work_in_both_import_orders(order, optimized): + run_probe(EARLY_PROBE, order, optimized) + + +@pytest.mark.parametrize("optimized", [False, True]) +@pytest.mark.parametrize("order", [ + ("functions_orchestration_results", "app", "background_tasks"), + ("app", "functions_orchestration_results", "background_tasks"), + ("background_tasks", "functions_orchestration_results"), +]) +def test_real_web_and_scheduler_imports_are_unchanged(order, optimized): + run_probe(APP_PROBE, order, optimized) + + +def test_new_result_modules_do_not_add_reverse_owner_imports_even_locally(): + forbidden = { + "config", "functions_settings", "functions_saved_analysis", + "functions_generated_file_exports", "functions_artifact_publication", + } + for name in ("functions_orchestration_result_contracts", "functions_orchestration_results"): + tree = ast.parse((APP / f"{name}.py").read_text(encoding="utf-8")) + imports = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.extend(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imports.append(node.module or "") + assert not forbidden.intersection(imports) + assert not any(name.startswith("route_") for name in imports) diff --git a/functional_tests/test_orchestration_results.py b/functional_tests/test_orchestration_results.py new file mode 100644 index 000000000..d46604861 --- /dev/null +++ b/functional_tests/test_orchestration_results.py @@ -0,0 +1,547 @@ +# test_orchestration_results.py +""" +M1 authorized retained-result persistence, complete readers, and lifecycle fences. +Version: 0.261.125 +Implemented in: 0.261.125 + +Real contracts, facade, record trees, transport, source/screening checks and SDK- +formatted conditional batches run with external Cosmos/Blob/source I/O doubled. +""" + +from copy import deepcopy +from dataclasses import replace +import json + +import pytest + +from content_screening.contracts import ScreeningError +from functions_generated_file_exports import GeneratedFileExportRequest, build_generated_file_export +from functions_orchestration_result_contracts import ( + InputBinding, InputSpec, RecordColumn, ResultContractError, ResultRef, TaskResult, canonical_bytes, +) +from functions_orchestration_results import NamedOutput, ResultUnavailableError, SavedAnalysisRecordSource +from functions_workflow_result_store import ( + ANALYSIS_CONTROL_RECORD_TYPE, + ORCHESTRATION_RESULT_RECORD_TYPE, + AnalysisWorkUnitConflictError, + WorkflowResultIntegrityError, + WorkflowResultStorageUnavailableError, + _orchestration_identity, +) +from test_support.orchestration_results import COLUMNS, ROWS, ResultFixture, complete, native_analysis, source +from test_workflow_result_store import FakeBlob + + +def store_arguments(producer): + return producer.user_id, producer.conversation_id, producer.run_id, producer.step_id + + +def commit_modified_manifest(fixture, reference, change): + store = fixture.service.store + args = store_arguments(reference.producer) + manifest = store.load_committed_orchestration_result(*args, reference.manifest_sha256) + change(manifest) + stored = store.save_orchestration( + *args, manifest, guard_token="server-attempt-token", require_analysis_guard=True, + ) + store.commit_orchestration_result(*args, stored, guard_token="server-attempt-token") + return replace(reference, manifest_sha256=stored["sha256"]) + + +@pytest.mark.parametrize("blob", [False, True]) +def test_full_restart_and_second_consumer_keep_original_records_and_schema(blob): + fixture = ResultFixture(blob=blob) + original = deepcopy(ROWS) + task = fixture.save() + descriptor = json.loads(json.dumps(task.to_dict())) + restart = fixture.restart() + first = restart.open_result(TaskResult.from_dict(descriptor).output("findings")) + preview = first.preview(max_items=1) + all_rows = list(first.iter_records()) + second = fixture.restart().open_result(ResultRef.from_dict(task.output("findings").to_dict())) + second_rows = list(second.iter_records()) + assert first.kind == "records" + assert first.record_count == 3 + assert first.columns == COLUMNS + assert all_rows == second_rows == original + assert all_rows[0]["id"] == "001" and all_rows[-1]["id"] == "last" + assert list(all_rows[0]) == [column.name for column in COLUMNS] + assert preview["preview"] is True and preview["integrity_verified"] is False + assert preview["value"] == original[:1] and preview["truncated"] is True + assert {row["type"] for row in fixture.container.items.values()} == { + ANALYSIS_CONTROL_RECORD_TYPE, ORCHESTRATION_RESULT_RECORD_TYPE, + } + assert not any("role" in row or "is_generated_chat_artifact" in row for row in fixture.container.items.values()) + if blob: + assert all(name.startswith("orchestration-analysis-results/") for _, name in fixture.blobs.records) + + +def test_30000_record_stream_exceeds_checkpoint_limit_without_losing_last_record(): + fixture = ResultFixture(blob=True) + produced = [] + + def rows(): + produced.append("called") + for index in range(30000): + yield {"id": f"{index:05}", "amount": index, "enabled": False, "detail": f"retained-{index} " + "x" * 320} + + task = fixture.save(grounded=False, outputs=[NamedOutput("rows", "records-v1", rows(), complete(30000), COLUMNS)]) + reference = task.output("rows") + checkpoint = canonical_bytes(task.to_dict()) + reader = fixture.restart().open_result(TaskResult.from_dict(json.loads(checkpoint)).output("rows")) + count, first, last = 0, None, None + for record in reader.iter_records(): + if count == 0: + first = record + last = record + count += 1 + second_reader = fixture.restart().open_result(reference) + second_count = 0 + for record in second_reader.iter_records(): + second_count += 1 + assert produced == ["called"] + assert count == second_count == 30000 + assert first["id"] == "00000" and last["id"] == "29999" + assert last["detail"].startswith("retained-29999 ") + assert reference.size_bytes > 8 * 1024 * 1024 + assert len(checkpoint) < 4096 + assert b"retained-29999" not in checkpoint + assert max(len(row["data"]) for row in fixture.blobs.records.values()) <= 128 * 1024 + + +@pytest.mark.parametrize("kind", ["text-v1", "markdown-v1", "structured-v1"]) +def test_large_values_use_streamed_sections_and_bounded_previews(kind): + fixture = ResultFixture(blob=True) + text = "complete\u4e2d\u6587\n" * 600000 + value = text if kind != "structured-v1" else {"text": text, "last": "final-value"} + task = fixture.save(grounded=False, outputs=[NamedOutput("prepared", kind, value, complete(1))]) + reader = fixture.restart().open_result(task.output("prepared")) + preview = reader.preview(max_bytes=257) + assert preview["truncated"] is True and preview["preview"] is True + assert len(preview["value"].encode("utf-8")) <= 257 + read = reader.read_value if kind == "structured-v1" else reader.read_text + if reader.reference.size_bytes > 8 * 1024 * 1024: + with pytest.raises(ResultContractError, match="invalid"): + read() + if kind == "structured-v1": + rebuilt = json.loads(b"".join(reader.iter_value_bytes())) + else: + rebuilt = "".join(reader.iter_text()) + assert rebuilt == value + if kind != "structured-v1": + assert reader.character_count == len(text) + assert max(len(row["data"]) for row in fixture.blobs.records.values()) <= 128 * 1024 + + +@pytest.mark.parametrize("kind,value", [ + ("records-v1", []), ("text-v1", ""), ("markdown-v1", ""), ("structured-v1", None), +]) +def test_proven_empty_outputs_are_not_unavailable_or_default_success(kind, value): + fixture = ResultFixture() + count = 0 if kind == "records-v1" else 1 + task = fixture.save(grounded=False, outputs=[ + NamedOutput("empty", kind, value, complete(count), COLUMNS if kind == "records-v1" else ()), + ]) + reader = fixture.restart().open_result(task.output("empty")) + actual = list(reader.iter_records()) if kind == "records-v1" else ( + reader.read_text() if kind in {"text-v1", "markdown-v1"} else reader.read_value() + ) + assert actual == value + + +@pytest.mark.parametrize("field,value", [ + ("user_id", "foreign"), ("conversation_id", "foreign-conversation"), ("run_id", "foreign-run"), + ("step_id", "missing"), ("capability_id", "different"), ("attempt_index", 2), +]) +def test_result_producer_owner_scope_and_attempt_mismatch_fail(field, value): + fixture = ResultFixture() + reference = fixture.save().output("findings") + forged = replace(reference, producer=replace(reference.producer, **{field: value})) + with pytest.raises((ResultUnavailableError, ResultContractError)): + fixture.service.open_result(forged) + + +@pytest.mark.parametrize("change", ["deleted", "revoked", "scope", "revision", "version", "screening"]) +def test_current_source_access_is_checked_on_consumption_not_only_save(change): + fixture = ResultFixture() + reference = fixture.save().output("findings") + reader = fixture.service.open_result(reference) + if change == "deleted": + fixture.sources.clear() + elif change == "revoked": + fixture.denied.add("document-1") + elif change == "scope": + fixture.sources["document-1"].update(scope="group", scope_id="foreign-group") + elif change == "revision": + fixture.sources["document-1"]["source_revision"] = "revision-2" + elif change == "version": + fixture.sources["document-1"]["source_version"] = 2 + else: + fixture.held.add("document-1") + with pytest.raises((PermissionError, ScreeningError)): + list(reader.iter_records()) + with pytest.raises((PermissionError, ScreeningError)): + fixture.restart().open_result(reference) + + +def test_explicit_historical_snapshot_never_refreshes_or_hides_changes(): + fixture = ResultFixture() + reference = fixture.save(source_policy="snapshot").output("findings") + fixture.sources["document-1"]["source_revision"] = "changed-revision" + reader = fixture.restart().open_result(reference) + metadata = reader.metadata() + rows = list(reader.iter_records()) + assert metadata["source_snapshot_changed"] is True + assert metadata["source_policy"] == "snapshot" and rows == ROWS + with pytest.raises(PermissionError): + fixture.service.open_result(reference, require_current_sources=True) + fixture.held.add("document-1") + with pytest.raises((PermissionError, ScreeningError)): + reader.recheck() + + +def test_current_source_content_digest_is_not_ignored(): + fixture = ResultFixture() + snapshot = {**source(), "content_sha256": "a" * 64} + fixture.sources["document-1"] = deepcopy(snapshot) + task = fixture.service.persist_task_result( + producer=fixture.producer, role="reason", status="complete", + outputs=[NamedOutput("result", "records-v1", ROWS, complete(3), COLUMNS)], + sources=[snapshot], origin="grounded", guard_token="server-attempt-token", + ) + fixture.sources["document-1"]["content_sha256"] = "b" * 64 + with pytest.raises(PermissionError): + fixture.service.open_result(task.output("result")) + + +@pytest.mark.parametrize("kind", ["source-set-v1", "evidence-set-v1"]) +@pytest.mark.parametrize("field,value", [ + ("source_version", 2), ("source_revision", "unretained-revision"), ("content_sha256", "a" * 64), +]) +def test_embedded_sources_must_match_the_exact_authorized_lineage_snapshot(kind, field, value): + fixture = ResultFixture() + forged = {**source(), field: value} + item = forged if kind == "source-set-v1" else { + "evidence_id": "evidence-1", "source": forged, "text": "A quote cannot change its source snapshot.", + } + with pytest.raises(ResultContractError) as error: + fixture.save(outputs=[NamedOutput("sources", kind, [item], complete(1))]) + assert error.value.code == "result_source_not_in_lineage" + assert not any(row.get("record_kind") == "final" for row in fixture.container.items.values()) + + +@pytest.mark.parametrize("change", ["conversation_deleted", "owner_changed", "run_deleted", "disabled", "missing", "capability", "attempt"]) +def test_current_producer_is_rechecked_after_open(change): + fixture = ResultFixture() + reader = fixture.service.open_result(fixture.save().output("findings")) + run = fixture.runs["run-1"] + if change == "conversation_deleted": + fixture.conversation["orchestration_deleted"] = True + elif change == "owner_changed": + fixture.conversation["user_id"] = "foreign" + elif change == "run_deleted": + run["checkpoints_deleted"] = True + elif change == "disabled": + run["plan"]["steps"][0]["enabled"] = False + elif change == "missing": + run["plan"]["steps"] = [] + elif change == "capability": + run["plan"]["steps"][0]["capability_id"] = "different" + else: + run["attempt_index"] = 2 + with pytest.raises(PermissionError): + reader.preview() + + +@pytest.mark.parametrize("status", ["pending", "invalid", "failed", "cancelled", "unavailable"]) +def test_unfinished_retained_data_is_not_a_success_shaped_input(status): + fixture = ResultFixture() + task = fixture.save(status=status, outputs=[ + NamedOutput("findings", "records-v1", [], complete(0, status=status, expected=3), COLUMNS), + ]) + assert task.status == status + with pytest.raises(ResultContractError): + fixture.service.open_result(task.output("findings"), allow_partial=True) + + +def test_partial_requires_explicit_consumption_and_cannot_be_promoted_by_a_child(): + fixture = ResultFixture() + task = fixture.save(status="partial", outputs=[ + NamedOutput("findings", "records-v1", ROWS[:2], complete(2, status="partial", expected=3), COLUMNS), + ]) + reference = task.output("findings") + with pytest.raises(ResultContractError): + fixture.service.open_result(reference) + reader = fixture.service.open_result(reference, allow_partial=True) + actual = list(reader.iter_records()) + assert actual == ROWS[:2] and reader.completeness.status == "partial" + consumer = fixture.consumer() + with pytest.raises(ResultContractError): + fixture.save( + grounded=False, producer=consumer, upstream=(reference,), allow_partial_inputs=True, + outputs=[NamedOutput("summary", "text-v1", "Only two of three rows.", complete(1))], + ) + + +def test_upstream_lineage_and_explicit_aliases_reauthorize_without_cross_attempt_rebinding(): + fixture = ResultFixture() + task = fixture.save() + consumer = fixture.consumer() + spec = InputSpec("data", InputBinding("analyze", "findings"), ("records-v1",)) + reader = fixture.service.resolve_input(spec, consumer=consumer, task_results={"analyze": task}) + actual = list(reader.iter_records()) + assert actual == ROWS + next_consumer = fixture.consumer(run_id="run-2", attempt_index=2) + with pytest.raises(ResultContractError): + fixture.service.resolve_input(spec, consumer=next_consumer, task_results={"analyze": task}) + alias = InputSpec("data", InputBinding(existing_result="saved_findings"), ("records-v1",)) + with pytest.raises(ResultContractError): + fixture.service.resolve_input(alias, consumer=next_consumer, task_results={}) + with pytest.raises(ResultContractError) as error: + fixture.service.resolve_input( + alias, consumer=task.producer, task_results={}, + existing_results={"saved_findings": task.output("findings")}, + ) + assert error.value.code == "result_binding_cycle" + reader = fixture.service.resolve_input( + alias, consumer=next_consumer, task_results={}, existing_results={"saved_findings": task.output("findings")}, + ) + actual = list(reader.iter_records()) + assert actual == ROWS + child = fixture.save( + grounded=False, producer=consumer, upstream=(task.output("findings"),), + outputs=[NamedOutput("draft", "text-v1", "Prepared from all original findings.", complete(1))], + ) + fixture.denied.add("document-1") + with pytest.raises(PermissionError): + fixture.restart().open_result(child.output("draft")) + + +def test_result_references_never_select_caller_supplied_storage(): + fixture = ResultFixture() + reference = fixture.save().output("findings") + with pytest.raises(ResultContractError): + ResultRef.from_dict({**reference.to_dict(), "storage": "blob", "blob_path": "foreign"}) + with pytest.raises(ResultContractError): + fixture.service.open_result(reference.to_dict()) + uncommitted = fixture.service.store.save_orchestration( + *store_arguments(reference.producer), {"private": "uncommitted"}, + guard_token="server-attempt-token", require_analysis_guard=True, + ) + with pytest.raises(WorkflowResultIntegrityError): + fixture.service.open_result(replace(reference, manifest_sha256=uncommitted["sha256"])) + + +def test_existing_exact_records_json_export_accepts_full_reader_without_a_new_profile(): + fixture = ResultFixture() + reference = fixture.save().output("findings") + reader = fixture.restart().open_result(reference) + with build_generated_file_export( + source=reader, export_request=GeneratedFileExportRequest("json"), + max_output_bytes=1024 * 1024, + ) as rendered: + data = rendered.file_content.read() + values = json.loads(data) + count = rendered.record_count + profile = rendered.profile + assert values == ROWS and count == 3 and profile == "exact_records_v1" + with pytest.raises(ValueError): + build_generated_file_export( + source=reader, export_request=GeneratedFileExportRequest("csv"), + max_output_bytes=1024 * 1024, + ) + + +def test_declared_content_digest_and_record_count_are_verified_not_only_transport(): + fixture = ResultFixture() + reference = fixture.save().output("findings") + corrupt = commit_modified_manifest( + fixture, reference, lambda manifest: manifest["outputs"]["findings"].update(content_sha256="0" * 64), + ) + corrupt = replace(corrupt, content_sha256="0" * 64) + reader = fixture.service.open_result(corrupt) + with pytest.raises(ResultContractError) as failure: + list(reader.iter_records()) + assert failure.value.code == "result_content_integrity" + corrupt_count = commit_modified_manifest( + fixture, reference, lambda manifest: manifest["outputs"]["findings"]["storage"].update(record_count=4), + ) + with pytest.raises(ResultContractError) as failure: + fixture.service.open_result(corrupt_count) + assert failure.value.code == "result_count_invalid" + + +def test_missing_final_record_fails_even_with_valid_transport_digests(): + fixture = ResultFixture() + reference = fixture.save().output("findings") + store = fixture.service.store + args = store_arguments(reference.producer) + + def drop_last(manifest): + output = manifest["outputs"]["findings"]["storage"] + index = store.load_orchestration(*args, output["result_ref"]) + leaf = index["value"]["children"][-1] + page = store.load_orchestration(*args, leaf["result_ref"]) + page["value"].pop() + leaf["result_ref"] = store.save_orchestration( + *args, page, guard_token="server-attempt-token", require_analysis_guard=True, + ) + output["result_ref"] = store.save_orchestration( + *args, index, guard_token="server-attempt-token", require_analysis_guard=True, + ) + + corrupt = commit_modified_manifest(fixture, reference, drop_last) + reader = fixture.service.open_result(corrupt) + with pytest.raises(ValueError, match="count"): + list(reader.iter_records()) + + +def test_unavailable_blob_backend_is_not_an_empty_or_preview_fallback(): + fixture = ResultFixture(blob=True) + reference = fixture.save().output("findings") + unavailable = fixture.restart(blobs=False) + with pytest.raises(WorkflowResultStorageUnavailableError): + unavailable.open_result(reference) + + +@pytest.mark.parametrize("blob", [False, True]) +def test_cleanup_keeps_tombstone_and_late_workers_cannot_recreate_results(blob): + fixture = ResultFixture(blob=blob) + reference = fixture.save().output("findings") + fixture.service.store.delete_orchestration_results("owner", "conversation-1", "run-1") + remaining = deepcopy(fixture.container.items) + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.save() + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.restart().open_result(reference) + assert fixture.container.items == remaining + assert len(remaining) == 1 and next(iter(remaining.values()))["deleted"] is True + if blob: + assert fixture.blobs.records == {} + + +def test_cancelled_fence_stops_writes_but_does_not_delete_completed_inputs(): + fixture = ResultFixture() + reference = fixture.save().output("findings") + identity = _orchestration_identity(*store_arguments(fixture.producer)) + fixture.service.store.cancel_analysis_attempt(identity, token="server-attempt-token") + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.save() + reader = fixture.restart().open_result(reference) + records = list(reader.iter_records()) + assert records == ROWS + + +def test_deletion_during_blob_upload_cannot_commit_or_recreate_payload(monkeypatch): + fixture = ResultFixture(blob=True) + identity = _orchestration_identity(*store_arguments(fixture.producer)) + original = FakeBlob.upload_blob + deleted = [] + + def upload(blob, **kwargs): + original(blob, **kwargs) + if not deleted: + fixture.service.store.fence_analysis_attempt(identity) + deleted.append(True) + + monkeypatch.setattr(FakeBlob, "upload_blob", upload) + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.save() + assert deleted == [True] + assert fixture.blobs.records == {} + assert all(row["record_kind"] == "lifecycle" for row in fixture.container.items.values()) + + +def test_cross_token_worker_cannot_write_after_owner_has_prepared(): + fixture = ResultFixture() + fixture.service.store.prepare_orchestration_result(*store_arguments(fixture.producer), guard_token="new-owner-token") + before = deepcopy(fixture.container.items) + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.save() + assert fixture.container.items == before + + +def test_evidence_sources_and_comparison_keep_distinct_full_kinds(): + fixture = ResultFixture() + fixture.sources["document-2"] = source("document-2") + sources = [source(), source("document-2")] + value = { + "left_document_id": "document-1", "right_document_ids": ["document-2"], + "items": [{"right_document_id": "document-2", "right_document_name": "Second source", "text": "Prepared differences."}], + "failed_document_ids": [], + } + task = fixture.service.persist_task_result( + producer=fixture.producer, role="reason", status="complete", sources=sources, + origin="grounded", guard_token="server-attempt-token", + outputs=[ + NamedOutput("sources", "source-set-v1", sources, complete(2)), + NamedOutput("evidence", "evidence-set-v1", [{"evidence_id": "e1", "source": source(), "text": "A source excerpt."}], complete(1)), + NamedOutput("comparison", "comparison-v1", value, complete(1)), + ], + ) + restarted = fixture.restart() + evidence = list(restarted.open_result(task.output("evidence")).iter_items()) + compared = restarted.open_result(task.output("comparison")).read_value() + saved_sources = list(restarted.open_result(task.output("sources")).iter_items()) + assert evidence[0]["text"] == "A source excerpt." + assert compared == value and saved_sources == sources + with pytest.raises(ResultContractError): + list(restarted.open_result(task.output("evidence")).iter_records()) + + +def test_nonempty_comparison_error_prose_cannot_claim_a_complete_comparison(): + fixture = ResultFixture() + fixture.sources["document-2"] = source("document-2") + comparison = { + "left_document_id": "document-1", "right_document_ids": ["document-2"], + "items": [], "failed_document_ids": ["document-2"], + } + with pytest.raises(ResultContractError): + fixture.service.persist_task_result( + producer=fixture.producer, role="reason", status="complete", sources=list(fixture.sources.values()), + origin="grounded", guard_token="server-attempt-token", + outputs=[NamedOutput("comparison", "comparison-v1", comparison, complete(1))], + ) + + +def test_saved_analyze_adapter_uses_actual_saved_reader_preserving_evidence_and_sources(): + # The owning service is deliberately imported at this integration boundary, + # never by the lower-level orchestration result modules. + from functions_saved_analysis import load_orchestration_analysis_input, save_orchestration_analysis + + fixture = ResultFixture() + native = native_analysis(150) + store = fixture.service.store + + def authorize(user_id, binding): + fixture.service.access.authorize_producer(fixture.producer) + if user_id != fixture.producer.user_id or binding != { + "kind": "orchestration", "user_id": "owner", "conversation_id": "conversation-1", + "run_id": "run-1", "step_id": "analyze", + }: + raise PermissionError("Unexpected saved Analyze producer.") + + def save_result(user_id, conversation_id, run_id, step_id, value, *, settings): + return store.save_orchestration(user_id, conversation_id, run_id, step_id, value) + + descriptor = save_orchestration_analysis( + {"analysis_result": native}, **dict(zip(("user_id", "conversation_id", "run_id", "step_id"), store_arguments(fixture.producer))), + settings={}, authorize_run=authorize, source_resolver=fixture.resolve, save_result=save_result, + ) + restarted = fixture.restart() + reader, _ = load_orchestration_analysis_input( + "owner", descriptor, authorize_run=authorize, load_result=restarted.store.load_orchestration, + source_resolver=fixture.resolve, bounded=True, + ) + adapter = SavedAnalysisRecordSource(reader, columns=(RecordColumn("finding", "string"), RecordColumn("detail", "string"))) + values = list(adapter.iter_records()) + units = list(adapter.iter_units()) + metadata = adapter.metadata() + assert len(values) == 150 and values[-1]["finding"] == "Complete finding 149" + assert units[-1]["record"]["record_id"] == "record-149" + assert units[-1]["evidence"][0]["quote"] == "Source quote 149" + assert metadata["sources"] == [source()] and metadata["coverage"] == native["coverage"] + assert "original_sources_reanalyzed" in metadata and metadata["original_sources_reanalyzed"] is False + fixture.denied.add("document-1") + with pytest.raises(PermissionError): + list(adapter.iter_records()) diff --git a/functional_tests/test_support/orchestration_results.py b/functional_tests/test_support/orchestration_results.py new file mode 100644 index 000000000..5783e4c0d --- /dev/null +++ b/functional_tests/test_support/orchestration_results.py @@ -0,0 +1,168 @@ +# orchestration_results.py +""" +External-I/O doubles and exact foundation fixtures. +Version: 0.261.125 +Implemented in: 0.261.125 +""" + +from copy import deepcopy +from dataclasses import replace + +from functions_orchestration_result_contracts import Completeness, Coverage, ProducerIdentity, RecordColumn +from functions_orchestration_results import NamedOutput, OrchestrationResultAccess, OrchestrationResults +from functions_workflow_result_store import WorkflowResultStore +from test_support.orchestration_revisions import AtomicMemoryContainer +from test_workflow_result_store import FakeBlobService + + +COLUMNS = ( + RecordColumn("id", "string"), + RecordColumn("amount", "number", nullable=True), + RecordColumn("enabled", "boolean"), + RecordColumn("detail", "json", nullable=True), +) +ROWS = [ + {"id": "001", "amount": 0, "enabled": False, "detail": {"text": "first\nline", "nested": [None, True]}}, + {"id": "=SUM(A1:A2)", "amount": 1.25, "enabled": True, "detail": "\u4e2d\u6587 caf\u00e9"}, + {"id": "last", "amount": None, "enabled": False, "detail": ["last", {"value": 7}]}, +] +SOURCE_FREE_VALUE = {"version": 1, "id": "007", "enabled": False, "nullable": None, "values": [1, "1", True]} +TABULAR_FOREGROUND = {"status": "completed", "kind": "records", "value": ROWS, "source_row_count": 3} +TABULAR_DURABLE = {"status": "running", "run_id": "native-job-1"} + + +def source(document_id="document-1"): + return { + "document_id": document_id, "scope": "personal", "scope_id": "owner", + "source_version": 1, "source_revision": "revision-1", + } + + +def complete(count, *, status="complete", expected=None): + return Completeness( + status, count if expected is None else expected, count, + Coverage(1, 1 if status == "complete" else 0, "work_units"), + {"complete": "valid", "partial": "partial", "invalid": "invalid", "pending": "pending"}.get(status, "not_validated"), + ("producer_schema",), () if status == "complete" else ("Only the retained subset is available.",), + ) + + +def native_analysis(count=3): + return { + "analysis_result_version": "analyze-final-v1", + "analysis_reply": "Complete narrative findings, not a published report.", + "source_manifest": [{**source(), "authorization_status": "authorized"}], + "authoritative_result": {"kind": "records", "value": [ + { + "record_id": f"record-{index}", "document_id": "document-1", + "values": {"finding": f"Complete finding {index}", "detail": "retained detail " * 30}, + "evidence_refs": [f"evidence-{index}"], + } for index in range(count) + ]}, + "analysis_evidence": [ + {"evidence_id": f"evidence-{index}", "document_id": "document-1", "quote": f"Source quote {index}"} + for index in range(count) + ], + "analysis_validation": {"status": "valid", "checks": ["record_shape"], "limitations": []}, + "coverage": {"documents": [{"document_id": "document-1", "total_windows": 1, "processed_windows": 1}]}, + } + + +class ResultContainer(AtomicMemoryContainer): + """Reuse the transactional fake with the shared result store's cleanup queries.""" + + def __init__(self): + super().__init__("run_id") + + def delete_item(self, item, partition_key, **kwargs): + if "etag" not in kwargs: + kwargs["etag"] = self.read_item(item, partition_key)["_etag"] + return super().delete_item(item, partition_key, **kwargs) + + def query_items(self, query, parameters=None, partition_key=None, **kwargs): + if "@record_type" not in query: + return super().query_items(query, parameters, partition_key, **kwargs) + values = {entry["name"][1:]: entry["value"] for entry in parameters} + self.queries.append((query, deepcopy(parameters), partition_key)) + return [ + deepcopy(row) for (partition, _), row in self.items.items() + if (partition_key is None or partition == partition_key) + and all( + row.get(name) == value for name, value in values.items() if name != "record_type" + ) + and row.get("type") == values["record_type"] and row.get("item_type") == values["record_type"] + ] + + +class ResultFixture: + def __init__(self, *, blob=False, max_bytes=32 * 1024 * 1024): + self.container = ResultContainer() + self.blobs = FakeBlobService() if blob else None + self.max_bytes = max_bytes + self.conversation = {"id": "conversation-1", "user_id": "owner"} + self.runs = {} + self.sources = {"document-1": source()} + self.held = set() + self.denied = set() + self.source_reads = [] + self.producer = ProducerIdentity("owner", "conversation-1", "run-1", 1, "analyze", "document_analyze", "analyze-final-v1") + self.add_producer(self.producer) + self.service = self.restart() + + def add_producer(self, producer): + run = self.runs.setdefault(producer.run_id, { + "id": producer.run_id, "user_id": producer.user_id, "conversation_id": producer.conversation_id, + "attempt_index": producer.attempt_index, "status": "running", "plan": {"steps": []}, + }) + run["plan"]["steps"].append({ + "step_id": producer.step_id, "capability_id": producer.capability_id, "enabled": True, + }) + + def consumer(self, **changes): + producer = replace(self.producer, step_id="consume", capability_id="compose", contract_version="compose-v1", **changes) + self.add_producer(producer) + return producer + + def resolve(self, ids, **scope): + self.source_reads.append((list(ids), deepcopy(scope))) + return [ + { + **deepcopy(self.sources.get(document_id) or source(document_id)), + "authorization_status": "authorized" if document_id in self.sources and document_id not in self.denied else "unresolved", + } + for document_id in ids + ] + + def metadata(self, document_id, user_id, group_id=None, public_workspace_id=None): + if user_id != "owner" or document_id not in self.sources or document_id in self.denied: + raise PermissionError("Fixture source access denied.") + snapshot = self.sources[document_id] + if group_id != (snapshot["scope_id"] if snapshot["scope"] == "group" else None): + raise PermissionError("Fixture group access denied.") + if public_workspace_id != (snapshot["scope_id"] if snapshot["scope"] == "public" else None): + raise PermissionError("Fixture public workspace access denied.") + document = {"id": document_id, "user_id": user_id, "version": snapshot["source_version"]} + if document_id in self.held: + document["content_screening"] = {"state": "pending_review"} + return document + + def restart(self, *, blobs=True): + store = WorkflowResultStore( + self.container, self.blobs if blobs else None, "private-results", + max_size_bytes=self.max_bytes, + ) + access = OrchestrationResultAccess( + user_id="owner", conversation_id="conversation-1", + read_conversation=lambda conversation_id: deepcopy(self.conversation), + read_run=lambda run_id: deepcopy(self.runs.get(run_id)), + source_resolver=self.resolve, source_metadata_reader=self.metadata, + ) + return OrchestrationResults(store, access) + + def save(self, *, outputs=None, producer=None, status="complete", grounded=True, **kwargs): + return self.service.persist_task_result( + producer=producer or self.producer, role="reason", status=status, + outputs=outputs if outputs is not None else [NamedOutput("findings", "records-v1", deepcopy(ROWS), complete(3), COLUMNS)], + sources=[source()] if grounded else [], origin="grounded" if grounded or kwargs.get("upstream") else "generated", + guard_token="server-attempt-token", **kwargs, + ) From b43164d97143109bc342d01c93123b179c6c81e2 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Mon, 21 Sep 2026 16:44:31 -0400 Subject: [PATCH 2/2] Fix orchestration result recovery and external lineage Preserve resumed native attempt guards, atomically bind committed results to exact producer/input receipts, and admit bounded external provenance with current access callbacks. Preserve v1 result bytes and checkpoint fingerprints; leave version ownership and runtime/export integration to the parent. Refs #1509 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...unctions_orchestration_result_contracts.py | 49 +++ .../functions_orchestration_results.py | 181 +++++++- .../functions_workflow_result_store.py | 104 ++++- .../ORCHESTRATION_RENDERING_HARNESS.md | 165 ++++++- ...RCHESTRATION_FOUNDATION_INTEGRATION_FIX.md | 77 ++++ .../test_orchestration_external_results.py | 273 ++++++++++++ .../test_orchestration_result_imports.py | 64 ++- .../test_orchestration_result_recovery.py | 412 ++++++++++++++++++ 8 files changed, 1300 insertions(+), 25 deletions(-) create mode 100644 docs/explanation/fixes/ORCHESTRATION_FOUNDATION_INTEGRATION_FIX.md create mode 100644 functional_tests/test_orchestration_external_results.py create mode 100644 functional_tests/test_orchestration_result_recovery.py diff --git a/application/single_app/functions_orchestration_result_contracts.py b/application/single_app/functions_orchestration_result_contracts.py index 7c9db570a..806022439 100644 --- a/application/single_app/functions_orchestration_result_contracts.py +++ b/application/single_app/functions_orchestration_result_contracts.py @@ -13,6 +13,12 @@ RESULT_REF_VERSION = "orchestration-result-ref-v1" INPUT_BINDING_VERSION = "orchestration-input-binding-v1" COMPLETENESS_VERSION = "orchestration-completeness-v1" +RESULT_MANIFEST_VERSION = "orchestration-result-manifest-v2" +RESULT_RECEIPT_VERSION = "orchestration-result-receipt-v1" +EXTERNAL_SOURCE_VERSION = "orchestration-external-source-v1" +EXTERNAL_LINEAGE_VERSION = "orchestration-lineage-v2" +EXTERNAL_SOURCE_TYPES = frozenset({"web", "url", "deep_research", "agent", "action", "fact_memory"}) +MAX_EXTERNAL_SOURCES = 64 RESULT_KINDS = frozenset({ "records-v1", "text-v1", "markdown-v1", "structured-v1", "evidence-set-v1", "source-set-v1", "comparison-v1", @@ -27,6 +33,7 @@ _NAME = re.compile(r"[A-Za-z][A-Za-z0-9_-]{0,63}\Z") _DIGEST = re.compile(r"[a-f0-9]{64}\Z") _COLUMN_TYPES = frozenset({"string", "integer", "number", "boolean", "object", "array", "json"}) +_OPAQUE_REFERENCE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,255}\Z") class ResultContractError(ValueError): @@ -173,6 +180,48 @@ def from_dict(cls, value): return cls(**cls._values(value)) +def result_receipt_binding(producer, input_fingerprint): + if type(producer) is not ProducerIdentity: + raise ResultContractError("result_producer_invalid") + digest(input_fingerprint) + return { + "version": RESULT_RECEIPT_VERSION, "producer": producer.to_dict(), + "input_fingerprint": input_fingerprint, + } + + +@dataclass(frozen=True) +class ExternalSourceRef(_Contract): + source_type: str + capability_id: str + reference_id: str + audience: str + content_sha256: str | None = None + source_revision: str | None = None + VERSION = EXTERNAL_SOURCE_VERSION + + def __post_init__(self): + choice(self.source_type, EXTERNAL_SOURCE_TYPES) + for value in (self.capability_id, self.reference_id, self.audience): + if type(value) is not str or _OPAQUE_REFERENCE.fullmatch(value) is None: + raise ResultContractError("result_external_identity_invalid") + if self.content_sha256 is None and self.source_revision is None: + raise ResultContractError("result_external_snapshot_required") + if self.content_sha256 is not None: + digest(self.content_sha256) + if self.source_revision is not None: + identifier(self.source_revision, limit=256) + if any(ord(character) < 32 or ord(character) == 127 for character in self.source_revision): + raise ResultContractError("result_external_snapshot_invalid") + + @classmethod + def from_dict(cls, value): + return cls(**cls._values(value)) + + def identity(self): + return self.source_type, self.capability_id, self.reference_id, self.audience + + def validate_result_role(producer, role): if type(producer) is not ProducerIdentity: raise ResultContractError("result_producer_invalid") diff --git a/application/single_app/functions_orchestration_results.py b/application/single_app/functions_orchestration_results.py index 1051ba6f4..014686386 100644 --- a/application/single_app/functions_orchestration_results.py +++ b/application/single_app/functions_orchestration_results.py @@ -14,12 +14,16 @@ from content_screening.access import assert_evidence_available from functions_analysis_access import analysis_source_snapshot, authorize_analysis_sources from functions_orchestration_result_contracts import ( + EXTERNAL_LINEAGE_VERSION, + MAX_EXTERNAL_SOURCES, MAX_DESCRIPTOR_BYTES, MAX_OUTPUTS, RESULT_KINDS, + RESULT_MANIFEST_VERSION, RESULT_STATES, TASK_RESULT_VERSION, Completeness, + ExternalSourceRef, InputSpec, ProducerIdentity, RecordColumn, @@ -28,6 +32,7 @@ TaskResult, canonical_bytes, choice, + digest, identifier, integer, output_name, @@ -94,18 +99,28 @@ class OrchestrationResultAccess: def __init__( self, *, user_id, conversation_id, read_conversation, read_run, source_resolver=None, source_metadata_reader=None, + external_source_catalog=None, external_source_authorizer=None, ): self.user_id = identifier(user_id) self.conversation_id = identifier(conversation_id) if not callable(read_conversation) or not callable(read_run): raise ResultContractError("result_access_reader_required") - for callback in (source_resolver, source_metadata_reader): + for callback in (source_resolver, source_metadata_reader, external_source_authorizer): if callback is not None and not callable(callback): raise ResultContractError("result_access_reader_required") self.read_conversation = read_conversation self.read_run = read_run self.source_resolver = source_resolver self.source_metadata_reader = source_metadata_reader + catalog = {} if external_source_catalog is None else external_source_catalog + if type(catalog) is not dict or len(catalog) > MAX_EXTERNAL_SOURCES: + raise ResultContractError("result_external_catalog_invalid") + for alias, reference in catalog.items(): + output_name(alias) + if type(reference) is not ExternalSourceRef: + raise ResultContractError("result_external_reference_untrusted") + self.external_source_catalog = dict(catalog) + self.external_source_authorizer = external_source_authorizer def authorize_producer(self, producer, *, for_write=False): if ( @@ -177,6 +192,39 @@ def resolve(document_ids, **scope): ) return {**checked, "source_snapshot_changed": checked["source_snapshot_changed"] or digest_changed} + def admit_external_sources(self, aliases): + if type(aliases) not in (list, tuple) or len(aliases) > MAX_EXTERNAL_SOURCES: + raise ResultContractError("result_external_catalog_invalid") + bindings = [] + for alias in aliases: + output_name(alias) + reference = self.external_source_catalog.get(alias) + if type(reference) is not ExternalSourceRef: + raise ResultContractError("result_external_reference_untrusted") + bindings.append({"alias": alias, "reference": reference.to_dict()}) + _external_bindings(bindings) + return bindings + + def authorize_external_sources(self, producer, bindings, *, require_snapshot): + references = _external_bindings(bindings) + if references and self.external_source_authorizer is None: + raise ResultUnavailableError("result_external_authorizer_required") + changed = False + for reference in references: + current = self.external_source_authorizer( + reference, producer=producer, user_id=self.user_id, conversation_id=self.conversation_id, + ) + if type(current) is not ExternalSourceRef or current.identity() != reference.identity(): + raise ResultUnavailableError("result_external_source_unavailable") + snapshot_changed = any( + getattr(reference, field) is not None and getattr(current, field) != getattr(reference, field) + for field in ("content_sha256", "source_revision") + ) + if snapshot_changed and require_snapshot: + raise ResultUnavailableError("result_external_snapshot_changed") + changed = changed or snapshot_changed + return changed, set(references) + def _producer_arguments(producer): return (producer.user_id, producer.conversation_id, producer.run_id, producer.step_id) @@ -196,8 +244,34 @@ def _sources(value): return snapshots +def _external_bindings(value): + if type(value) is not list or len(value) > MAX_EXTERNAL_SOURCES: + raise ResultContractError("result_external_catalog_invalid") + aliases, identities, references = set(), set(), [] + for binding in value: + if type(binding) is not dict or set(binding) != {"alias", "reference"}: + raise ResultContractError("result_external_catalog_invalid") + alias = output_name(binding["alias"]) + reference = ExternalSourceRef.from_dict(binding["reference"]) + if alias in aliases or reference.identity() in identities: + raise ResultContractError("result_external_duplicate_binding") + aliases.add(alias) + identities.add(reference.identity()) + references.append(reference) + return tuple(references) + + def _lineage(value): - if type(value) is not dict or set(value) != _LINEAGE_FIELDS: + if type(value) is not dict: + raise ResultContractError("result_lineage_invalid") + expected_fields = _LINEAGE_FIELDS + external = () + if value.get("version") == EXTERNAL_LINEAGE_VERSION: + expected_fields = expected_fields | {"version", "external_sources"} + external = _external_bindings(value.get("external_sources")) + if not external: + raise ResultContractError("result_lineage_invalid") + if set(value) != expected_fields: raise ResultContractError("result_lineage_invalid") choice(value["origin"], {"generated", "grounded"}) choice(value["source_policy"], {"current", "snapshot"}) @@ -210,7 +284,7 @@ def _lineage(value): references = tuple(ResultRef.from_dict(item) for item in parents) if len(set(references)) != len(references): raise ResultContractError("result_lineage_invalid") - if (value["origin"] == "generated") != (not value["sources"] and not references): + if (value["origin"] == "generated") != (not value["sources"] and not references and not external): raise ResultContractError("result_lineage_invalid") if len(canonical_bytes(value)) > MAX_DESCRIPTOR_BYTES: raise ResultContractError("result_descriptor_too_large") @@ -244,18 +318,38 @@ def _reference(producer, name, output, manifest_sha256): def _task_result(manifest, manifest_sha256): - if type(manifest) is not dict or set(manifest) != _MANIFEST_FIELDS or manifest["version"] != TASK_RESULT_VERSION: + if type(manifest) is not dict: + raise ResultContractError("result_manifest_invalid") + version = manifest.get("version") + expected_fields = _MANIFEST_FIELDS + if version == RESULT_MANIFEST_VERSION: + expected_fields = expected_fields | {"input_fingerprint", "output_order"} + if manifest.get("input_fingerprint") is not None: + digest(manifest["input_fingerprint"]) + elif version != TASK_RESULT_VERSION: + raise ResultContractError("result_manifest_invalid") + if set(manifest) != expected_fields: raise ResultContractError("result_manifest_invalid") if len(canonical_bytes(manifest)) > MAX_DESCRIPTOR_BYTES: raise ResultContractError("result_descriptor_too_large") producer = ProducerIdentity.from_dict(manifest["producer"]) _lineage(manifest["lineage"]) + if version == TASK_RESULT_VERSION and "version" in manifest["lineage"]: + raise ResultContractError("result_manifest_invalid") outputs = manifest["outputs"] if type(outputs) is not dict or not 1 <= len(outputs) <= MAX_OUTPUTS: raise ResultContractError("result_manifest_invalid") + order = list(outputs) + if version == RESULT_MANIFEST_VERSION: + order = manifest["output_order"] + if ( + type(order) is not list or any(type(name) is not str for name in order) + or len(order) != len(outputs) or set(order) != set(outputs) + ): + raise ResultContractError("result_manifest_invalid") return TaskResult( producer, manifest["role"], manifest["status"], - tuple(_reference(producer, name, output, manifest_sha256) for name, output in outputs.items()), + tuple(_reference(producer, name, outputs[name], manifest_sha256) for name in order), ) @@ -353,7 +447,10 @@ def _authorize_lineage(self, producer, lineage, *, for_write=False, force_curren source_snapshots = { _source_key(source): {canonical_bytes(source)} for source in lineage["sources"] } - changed = checked["source_snapshot_changed"] + external_changed, external_sources = self.access.authorize_external_sources( + producer, lineage.get("external_sources", []), require_snapshot=current, + ) + changed = checked["source_snapshot_changed"] or external_changed active = set() if active is None else active visited = {} if visited is None else visited for parent in parents: @@ -374,15 +471,19 @@ def _authorize_lineage(self, producer, lineage, *, for_write=False, force_curren ) finally: active.remove(key) - parent_changed, parent_snapshots = visited[key] + parent_changed, parent_snapshots, parent_external = visited[key] changed = changed or parent_changed for source_key, snapshots in parent_snapshots.items(): source_snapshots.setdefault(source_key, set()).update(snapshots) - return changed, source_snapshots + external_sources.update(parent_external) + if len(external_sources) > MAX_LINEAGE_RESULTS: + raise ResultContractError("result_lineage_limit") + return changed, source_snapshots, external_sources def persist_task_result( self, *, producer, role, status, outputs, sources, origin, guard_token, - upstream=(), source_policy="current", allow_partial_inputs=False, + upstream=(), source_policy="current", allow_partial_inputs=False, input_fingerprint=None, + external_sources=(), ): """Retain final data/diagnostics and return only immutable named descriptors. @@ -399,6 +500,10 @@ def persist_task_result( raise ResultContractError("result_outputs_invalid") if any(type(output) is not NamedOutput for output in outputs): raise ResultContractError("result_outputs_invalid") + if input_fingerprint is not None: + digest(input_fingerprint) + if status not in {"complete", "partial"}: + raise ResultContractError("result_receipt_not_terminal") if len({output.name for output in outputs}) != len(outputs): raise ResultContractError("result_duplicate_output") if type(upstream) not in (list, tuple) or any(type(parent) is not ResultRef for parent in upstream): @@ -407,12 +512,15 @@ def persist_task_result( "origin": origin, "source_policy": source_policy, "sources": deepcopy(sources), "upstream": [parent.to_dict() for parent in upstream], "allow_partial_inputs": allow_partial_inputs, } + external_bindings = self.access.admit_external_sources(external_sources) + if external_bindings: + lineage.update(version=EXTERNAL_LINEAGE_VERSION, external_sources=external_bindings) parents = _lineage(lineage) if any(parent.completeness.status == "partial" for parent in parents) and any( output.completeness.status == "complete" for output in outputs ): raise ResultContractError("result_partial_promoted") - _, source_snapshots = self._authorize_lineage(producer, lineage, for_write=True) + _, source_snapshots, _ = self._authorize_lineage(producer, lineage, for_write=True) if status == "complete" and any(output.completeness.status != "complete" for output in outputs): raise ResultContractError("result_incomplete") self.store.prepare_orchestration_result(*_producer_arguments(producer), guard_token=guard_token) @@ -491,16 +599,54 @@ def account(data): "version": TASK_RESULT_VERSION, "producer": producer.to_dict(), "role": role, "status": status, "lineage": lineage, "outputs": stored_outputs, } + if input_fingerprint is not None or external_bindings: + manifest.update( + version=RESULT_MANIFEST_VERSION, input_fingerprint=input_fingerprint, + output_order=list(stored_outputs), + ) encoded = canonical_bytes(manifest) result = _task_result(manifest, hashlib.sha256(encoded).hexdigest()) reference = budget.save_section(manifest, save, max_section_bytes=MAX_DESCRIPTOR_BYTES) if reference["sha256"] != result.outputs[0].manifest_sha256: raise ResultContractError("result_digest_invalid") self._authorize_lineage(producer, lineage, for_write=True) - self.store.commit_orchestration_result(*_producer_arguments(producer), reference, guard_token=guard_token) + receipt_options = ( + {"producer": producer, "input_fingerprint": input_fingerprint} if input_fingerprint is not None else {} + ) + self.store.commit_orchestration_result( + *_producer_arguments(producer), reference, guard_token=guard_token, **receipt_options, + ) self._authorize_lineage(producer, lineage, for_write=True) return result + def recover_task_result(self, *, producer, input_fingerprint): + """Recover one committed terminal result before replaying any producer work.""" + self.access.authorize_producer(producer) + digest(input_fingerprint) + receipt = self.store.load_orchestration_result_receipt(producer, input_fingerprint) + if receipt is None: + self.access.authorize_producer(producer) + return None + manifest_sha256, manifest = receipt + task = _task_result(manifest, manifest_sha256) + if task.producer != producer or task.status not in {"complete", "partial"}: + raise ResultContractError("result_receipt_invalid") + self._authorize_lineage(producer, manifest["lineage"]) + for reference in task.outputs: + if reference.completeness.status not in {"complete", "partial"}: + continue + reader = self.open_result(reference, allow_partial=True) + if reference.kind in _COLLECTION_KINDS: + values = reader.iter_items() + elif reference.kind in _TEXT_KINDS: + values = reader.iter_text() + else: + values = reader.iter_value_bytes() + for _ in values: + pass + self._authorize_lineage(producer, manifest["lineage"]) + return task + def open_result(self, reference, *, allow_partial=False, require_current_sources=False): if type(reference) is not ResultRef or type(require_current_sources) is not bool: raise ResultContractError("result_reference_untrusted") @@ -564,22 +710,31 @@ def __init__(self, service, reference, manifest, *, allow_partial, require_curre self.columns = reference.columns self.completeness = reference.completeness self._source_snapshots = {} + self._external_sources = set() self._changed = False def recheck(self): self.reference.completeness.require_readable(allow_partial=self._allow_partial) self._service._load_manifest(self.reference) - self._changed, self._source_snapshots = self._service._authorize_lineage( + self._changed, self._source_snapshots, self._external_sources = self._service._authorize_lineage( self.reference.producer, self._manifest["lineage"], force_current=self._require_current_sources, ) def metadata(self): self.recheck() - return { + metadata = { "reference": self.reference.to_dict(), "source_count": len(self._source_snapshots), "source_snapshot_changed": self._changed, "origin": self._manifest["lineage"]["origin"], "source_policy": self._manifest["lineage"]["source_policy"], } + if self._manifest["version"] == RESULT_MANIFEST_VERSION: + metadata["input_fingerprint"] = self._manifest["input_fingerprint"] + if self._external_sources: + metadata.update( + external_source_count=len({reference.identity() for reference in self._external_sources}), + external_sources=deepcopy(self._manifest["lineage"].get("external_sources", [])), + ) + return metadata def _collection(self): output = self._manifest["outputs"][self.reference.output_name] diff --git a/application/single_app/functions_workflow_result_store.py b/application/single_app/functions_workflow_result_store.py index 6654221ad..90c134986 100644 --- a/application/single_app/functions_workflow_result_store.py +++ b/application/single_app/functions_workflow_result_store.py @@ -47,6 +47,12 @@ from azure.cosmos.exceptions import CosmosResourceExistsError, CosmosResourceNotFoundError from azure.storage.blob import ContentSettings +from functions_orchestration_result_contracts import ( + RESULT_MANIFEST_VERSION, + RESULT_RECEIPT_VERSION, + canonical_digest, + result_receipt_binding, +) from functions_workflow_runtime_store import WorkflowRuntimeConflict from functions_workflow_identity import workflow_execution_id, workflow_node_identity @@ -69,6 +75,13 @@ class WorkflowResultIntegrityError(ValueError): """Stored identity, metadata, or payload does not match the expected result.""" +class OrchestrationResultConflictError(WorkflowResultIntegrityError): + code = "orchestration_result_commit_collision" + + def __init__(self): + super().__init__("A different result is already committed for this producer and input.") + + class WorkflowResultTooLargeError(ValueError): """The complete serialized result exceeds the configured artifact quota.""" @@ -458,12 +471,16 @@ def save_orchestration( def prepare_orchestration_result(self, user_id, conversation_id, run_id, step_id, *, guard_token): """Register an authorized producer using the existing execution/deletion fence.""" - return self.prepare_analysis_attempt( - _orchestration_identity(user_id, conversation_id, run_id, step_id), token=guard_token, - ) + identity = _orchestration_identity(user_id, conversation_id, run_id, step_id) + token = _identifier(guard_token) + guard = self._analysis_guard(identity, writable=True, token=token) + if guard is not None: + return {"binding": dict(identity)} + return self.prepare_analysis_attempt(identity, token=token) def commit_orchestration_result( self, user_id, conversation_id, run_id, step_id, reference, *, guard_token, + producer=None, input_fingerprint=None, ): """Commit a private digest-addressed manifest pointer, never a downloadable file.""" identity = _orchestration_identity(user_id, conversation_id, run_id, step_id) @@ -476,7 +493,59 @@ def commit_orchestration_result( "type": ANALYSIS_CONTROL_RECORD_TYPE, "item_type": ANALYSIS_CONTROL_RECORD_TYPE, "record_kind": "final", "key": key, "reference": reference, } - self._write_analysis_record(identity, row, token=guard_token, immutable=True) + if producer is None and input_fingerprint is None: + self._write_analysis_record(identity, row, token=guard_token, immutable=True) + return + binding = result_receipt_binding(producer, input_fingerprint) + _require_fields(producer.to_dict(), { + key: identity[key] for key in ("user_id", "conversation_id", "run_id", "step_id") + }) + payload = self._load(identity, reference) + _require_fields(payload, { + "version": RESULT_MANIFEST_VERSION, + "producer": binding["producer"], "input_fingerprint": input_fingerprint, + }) + if payload.get("status") not in {"complete", "partial"}: + raise WorkflowResultIntegrityError("Only terminal results can have an orchestration receipt.") + receipt_key = f"{RESULT_RECEIPT_VERSION}:{canonical_digest(binding)}" + receipt = { + **identity, "id": _analysis_control_id(identity, "final", receipt_key), + "type": ANALYSIS_CONTROL_RECORD_TYPE, "item_type": ANALYSIS_CONTROL_RECORD_TYPE, + "record_kind": "final", "key": receipt_key, "binding": binding, "reference": reference, + } + self._write_analysis_record( + identity, receipt, token=guard_token, immutable=True, companions=(row,), + ) + + def load_orchestration_result_receipt(self, producer, input_fingerprint): + """Read one exact producer/input commit; only a missing receipt returns None.""" + binding = result_receipt_binding(producer, input_fingerprint) + identity = _orchestration_identity( + producer.user_id, producer.conversation_id, producer.run_id, producer.step_id, + ) + guard = self._analysis_guard(identity) + if guard is not None and guard.get("deleted"): + raise AnalysisWorkUnitConflictError("analysis_work_deleted") + key = f"{RESULT_RECEIPT_VERSION}:{canonical_digest(binding)}" + row = self._analysis_control(identity, "final", key) + if row is None: + guard = self._analysis_guard(identity) + if guard is not None and guard.get("deleted"): + raise AnalysisWorkUnitConflictError("analysis_work_deleted") + return None + self._analysis_guard(identity, required=True) + _require_fields(row, {"binding": binding}) + _require_fields(row["binding"]["producer"], binding["producer"]) + reference = _validate_reference(row.get("reference")) + commit_key = f"{ORCHESTRATION_RESULT_COMMIT_KEY}:{reference['sha256']}" + committed = self.read_analysis_checkpoint(identity, "final", commit_key) + _require_fields(committed, {"reference": reference}) + manifest = self._load(identity, reference) + _require_fields(manifest, { + "version": RESULT_MANIFEST_VERSION, + "producer": binding["producer"], "input_fingerprint": input_fingerprint, + }) + return reference["sha256"], manifest def load_committed_orchestration_result(self, user_id, conversation_id, run_id, step_id, manifest_sha256): """Resolve storage only from a producer-bound immutable server commit.""" @@ -626,8 +695,17 @@ def _analysis_control(self, identity, kind, key=""): }) return row - def _write_analysis_record(self, identity, record, *, token, immutable=False, previous=None): + def _write_analysis_record(self, identity, record, *, token, immutable=False, previous=None, companions=()): """Fence a bounded control/payload write in the existing run-items partition.""" + if companions: + if not immutable or previous is not None or type(companions) is not tuple or len(companions) != 1: + raise ValueError("A companion requires one immutable same-partition commit.") + _require_fields(companions[0], { + **identity, "type": ANALYSIS_CONTROL_RECORD_TYPE, "item_type": ANALYSIS_CONTROL_RECORD_TYPE, + "record_kind": "final", + }) + if companions[0]["id"] == record["id"]: + raise WorkflowResultIntegrityError("Result commit identities must be distinct.") for _ in range(8): guard = self._analysis_guard(identity, required=True, writable=True, token=token) replacement = {key: value for key, value in guard.items() if not key.startswith("_")} @@ -643,6 +721,7 @@ def _write_analysis_record(self, identity, record, *, token, immutable=False, pr if not previous.get("_etag"): raise WorkflowResultIntegrityError("Analysis claim is missing its conditional-write version.") operations.append(("replace", (record["id"], record), {"if_match_etag": previous["_etag"]})) + operations.extend(("create", (companion,)) for companion in companions) execution = _workflow_execution_guard(identity, self._workflow_execution) if execution is not None: operations = execution.fence_batch(operations) @@ -659,8 +738,19 @@ def _write_analysis_record(self, identity, record, *, token, immutable=False, pr if latest.get("_etag") == previous.get("_etag"): continue if immutable and exc.status_code == 409: - saved = self.container.read_item(item=record["id"], partition_key=identity["run_id"]) - _require_fields(saved, record) + for expected in (record, *companions): + try: + saved = self.container.read_item(item=expected["id"], partition_key=identity["run_id"]) + except CosmosResourceNotFoundError: + if not companions: + raise + raise WorkflowResultIntegrityError("The immutable result commit is incomplete.") from None + if companions and expected is record: + _require_fields(saved, {key: value for key, value in expected.items() if key != "reference"}) + _require_fields(saved["binding"]["producer"], record["binding"]["producer"]) + if _validate_reference(saved.get("reference")) != record["reference"]: + raise OrchestrationResultConflictError() + _require_fields(saved, expected) self._analysis_guard(identity, required=True, writable=True, token=token) _workflow_execution_guard(identity, self._workflow_execution) return diff --git a/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md b/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md index 13ac35f87..7f92a2bd9 100644 --- a/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md +++ b/docs/explanation/features/ORCHESTRATION_RENDERING_HARNESS.md @@ -176,6 +176,8 @@ OrchestrationResultAccess( read_run, source_resolver=None, source_metadata_reader=None, + external_source_catalog=None, + external_source_authorizer=None, ) OrchestrationResults(store, access, *, max_result_bytes=None) ``` @@ -188,6 +190,7 @@ The callbacks are server-owned: | `read_run(run_id)` | Return the current run record, including owner, conversation, attempt, current plan steps, cancellation and deletion state. | | `source_resolver(document_ids, **scope)` | Implement the existing `resolve_authorized_source_manifest` protocol; current access/scope/version/revision information must come from authorized source metadata, not the request. | | `source_metadata_reader(document_id, user_id, group_id=None, public_workspace_id=None)` | Enforce the actor's current source/workspace access and return current screening metadata using the existing screening reader protocol. | +| `external_source_authorizer(reference, *, producer, user_id, conversation_id)` | Return a current `ExternalSourceRef` only after checking current capability, integration/resource access, and the original audience. Required for every external source type, including public web results. | Source-free results need no source callbacks. Grounded results fail closed if a required callback or source is unavailable. There is no application-settings or @@ -208,8 +211,14 @@ service.persist_task_result( upstream=(), source_policy="current", allow_partial_inputs=False, + input_fingerprint=None, + external_sources=(), ) # -> TaskResult +service.recover_task_result( + *, producer, input_fingerprint, +) # -> TaskResult | None + service.open_result( reference, *, allow_partial=False, require_current_sources=False, ) # -> OrchestrationResultReader @@ -232,10 +241,12 @@ store.prepare_orchestration_result( ) store.commit_orchestration_result( user_id, conversation_id, run_id, step_id, reference, *, guard_token, + producer=None, input_fingerprint=None, ) store.load_committed_orchestration_result( user_id, conversation_id, run_id, step_id, manifest_sha256, ) +store.load_orchestration_result_receipt(producer, input_fingerprint) ``` Private sections reuse the existing orchestration result namespace, including @@ -244,6 +255,141 @@ name, not a claim that generic content satisfies Analyze's native schema. Only the final immutable commit makes a descriptor readable. A caller-provided digest for an uncommitted section cannot bypass that boundary. +### Native resume and committed-result recovery + +Generic saving shares the native step's **existing active token and resume +binding**. It does not call native preparation again with a default +`resume_from=None`, replace the original request/source digests, or start another +attempt. A missing guard is initialized normally; an existing guard must still +be writable by the owning token. Cancelled, deleted, superseded, or differently +tokened workers are rejected. + +An owning runtime may provide its server-computed, 64-hex `input_fingerprint` for +a terminal `complete` or `partial` task result. The store writes one deterministic +producer/input receipt and the digest commit in the **same lifecycle-CAS +transaction**. The key includes the complete `ProducerIdentity`, including +attempt and producer contract version, plus the input fingerprint. + +| Write/recovery condition | Result | +| --- | --- | +| Same exact identity and same committed digest | Idempotent success. | +| Same identity but a different result digest | `OrchestrationResultConflictError`, code `orchestration_result_commit_collision`; no replacement or latest-result selection. | +| No receipt for the exact producer/input identity | `recover_task_result()` returns `None`; other inputs/attempts are not scanned. | +| A receipt exists but its binding, commit, manifest or data is corrupt/unavailable | Explicit failure, not `None`, a preview, or permission to replay a model. | +| Cancellation/deletion wins the commit CAS | Neither receipt nor digest commit becomes visible. | +| Commit succeeds and the runtime stops before saving its checkpoint | The same authenticated lookup recovers the original result without needing the lost manifest digest. | + +Recovery reauthorizes the original producer, document sources/screening, +external sources and upstream lineage, then fully streams and verifies every +readable output before returning. It preserves output order, partial coverage +and nonreadable failure descriptors. Consumers still cannot bind a failed +output as data. Recovery performs retained-storage/metadata I/O, not model work, +source-content fetching, or reconstruction from previews. + +A completed result remains recoverable after a later sibling fails or the +original attempt is stopped. It retains its **original** run/attempt identity. +Cross-attempt use requires the normal explicit result alias; recovery does not +relabel the old result as a new producer. Pending native jobs stay in their +existing job/wait lifecycle and do not receive completed-result receipts. + +The optional private manifest version is +`orchestration-result-manifest-v2`, with `input_fingerprint` and `output_order`. +The fingerprint is nullable when only external lineage opts into this version. +The receipt binding version is `orchestration-result-receipt-v1`. Public +`TaskResult` and `ResultRef` wire schemas remain v1. With neither extension used, +the original v1 manifest and descriptor bytes remain unchanged; no old result is +silently assigned a new input identity. + +### External Gather provenance + +Web, URL, deep-research, agent, action, and Fact Memory results can carry explicit +external provenance rather than fake document IDs or a source-free label: + +```python +ExternalSourceRef( + source_type, + capability_id, + reference_id, + audience, + content_sha256=None, + source_revision=None, +) +``` + +This frozen descriptor has strict `to_dict()` / `from_dict()` methods and wire +version `orchestration-external-source-v1`. `source_type` is one of `web`, `url`, +`deep_research`, `agent`, `action`, or `fact_memory`. Capability, reference, and +audience are opaque ASCII identifiers of at most 256 characters, using letters, +digits, `_`, `.`, `:`, and `-`; they must start with a letter or digit. They are +not URLs, storage paths, credentials, callbacks or raw memory prompts. A bounded +source revision or SHA-256 digest is required. A revision may retain an ETag, +but cannot contain control characters or exceed 256 UTF-8 bytes. + +The owner supplies `external_source_catalog` as an explicitly authorized +`dict[str, ExternalSourceRef]`. `external_sources` passed to persistence is a +list/tuple of aliases from this catalog, not model-supplied descriptors. Catalogs +and direct bindings are limited to 64 entries. Aliases and exact external +resource identities cannot repeat in one result's bindings. + +The committed lineage uses `version="orchestration-lineage-v2"` and stores: + +```json +{ + "external_sources": [ + { + "alias": "admitted_source", + "reference": { + "version": "orchestration-external-source-v1", + "source_type": "web", + "capability_id": "web_search", + "reference_id": "server-owned-search-result", + "audience": "personal:original-owner", + "content_sha256": null, + "source_revision": "retained-revision-1" + } + } + ] +} +``` + +This is the external-binding portion of the lineage; its ordinary origin, +policy, document sources, upstream references and partial-input policy remain +required. A grounded lineage can contain documents, external bindings, upstream +results, or a combination. Generated content must contain none of them. + +Every external descriptor is reauthorized by the injected callback on saving, +reading, recovery and transitive reuse. Returning `True`, a dictionary or `None` +does not grant access. The callback must return a current typed descriptor or +raise. Its source type, capability, reference ID and audience must exactly match +the original. A source revision/digest change fails under `current`; explicit +`snapshot` access retains the original data and reports the change, without +relaxing current permission, capability or audience checks. + +Current here means **current authorized server resource metadata**, not a hidden +network refresh. Public display URLs may be retained in prepared content, but +are neither fetch instructions nor permission to use a tool. The callback must +enforce current integration/agent access or memory audience as applicable. This +does not expand Fact Memory across conversations or introduce global memory. + +Aliases and original descriptors are durable server manifest data, not a +Python-only cache. Reads after restart reauthorize those committed bindings +without requiring the old admission catalog. `metadata()["external_sources"]` +returns the **direct** persisted bindings so an authorized owner can rebuild its +catalog with `ExternalSourceRef.from_dict()`. New writes still require catalog +admission. Upstream references preserve their own producer-bound aliases; they +are not flattened into a potentially ambiguous global catalog. + +Existing `source_count` retains its document-source meaning. Optional +`external_source_count` includes distinct inherited external resources, and +`source_snapshot_changed` covers either source class. Traversal permits at most +256 retained external snapshots. Existing document-only v1 lineages are not +reinterpreted. + +External content uses existing text/Markdown, records or structured-value kinds +as appropriate. `evidence-set-v1` and `source-set-v1` remain document-scoped. +Adding provenance does not advertise a new renderer or make an external +integration call. + ### Access, lineage, and attempt boundaries Reuse is original-owner and same-conversation only. Every read checks the @@ -257,8 +403,8 @@ running run with no cancellation request or successor attempt. optional `content_sha256`. Cached authorization decisions and storage locators are not accepted as source snapshots. -`origin="generated"` requires no source/upstream lineage. `origin="grounded"` -requires actual sources or upstream results. Upstream references are recursively +`origin="generated"` requires no document/external/upstream lineage. +`origin="grounded"` requires actual sources or upstream results. Upstream references are recursively reauthorized, cycle-checked, and bounded. Their source snapshots remain part of the child's authorization boundary. @@ -356,6 +502,8 @@ count, distinct from UTF-8 byte size. Structured/comparison readers expose `read_value()` and streamed canonical JSON through `iter_value_bytes()`. `metadata()` returns the reference, distinct source count, origin, source policy, and source-change flag. +For a v2 manifest it also includes the optional input fingerprint. External +lineage metadata is described above; it does not expose executable fetch handles. **Consumers must exhaust a full iterator before accepting or publishing its result.** Final count/digest checks and the last access recheck run at exhaustion. @@ -383,6 +531,7 @@ preview need not be a parseable complete JSON value. | Comparison value | At most 8 MiB, including semantic count validation during streamed reads. | | Preview | At most 20 items / 16 KiB. | | Upstream references / traversed lineage results | 64 / 256. | +| External admission catalog / direct external bindings / retained external snapshots | 64 / 64 / 256. | | Aggregate retained bytes | `max_result_bytes`, defaulting to the injected store's `max_size_bytes`; sections and indexes share one budget. | Oversized records, manifests, or aggregate data fail rather than being shortened. @@ -424,6 +573,7 @@ The new regression suites are: ```powershell python -m pytest .\functional_tests\test_orchestration_result_contracts.py .\functional_tests\test_orchestration_results.py .\functional_tests\test_orchestration_result_imports.py -q +python -m pytest .\functional_tests\test_orchestration_result_recovery.py .\functional_tests\test_orchestration_external_results.py -q ``` They exercise real production contracts/transport with external I/O doubles: @@ -434,11 +584,22 @@ reuse by two readers after persistence/restart; a 30,000-row dataset exceeding source/producer/screening checks; explicit historical snapshots; stale tokens; cancel/delete races; no artifact publication; unchanged v1 checkpoint fingerprints; and the existing JSON export protocol. +The follow-up cases also cover real native resumed work-unit guards, atomic +receipt rollback/collision races, commit-before-checkpoint crashes, recovery +after sibling failure, complete readable-data verification, exact published-v1 +byte fingerprints, all external source types, lost audience/capability/resource +access, explicit historical external snapshots, and catalog restoration. Cold-import checks use fresh normal and optimized interpreters with network blocked. They cover both lower-level import orders, required storage operations and failure paths, and normal web and separate scheduler imports using real runtime modules rather than a fake `config`. +The early normal/optimized probes execute native resumed preparation, receipt +recovery and external-source reauthorization with explicit checks that cannot +disappear under `-O`. + +Root causes and before/after behavior for these integration seams are recorded in +[the foundation integration fix](../fixes/ORCHESTRATION_FOUNDATION_INTEGRATION_FIX.md). Focused existing regressions cover Analyze storage/source access, orchestration saved-result integration, checkpoint recovery/access, generated-file saved-record diff --git a/docs/explanation/fixes/ORCHESTRATION_FOUNDATION_INTEGRATION_FIX.md b/docs/explanation/fixes/ORCHESTRATION_FOUNDATION_INTEGRATION_FIX.md new file mode 100644 index 000000000..0260f9d64 --- /dev/null +++ b/docs/explanation/fixes/ORCHESTRATION_FOUNDATION_INTEGRATION_FIX.md @@ -0,0 +1,77 @@ +# Orchestration foundation integration fixes + +**Version: 0.261.125** + +Fixed in version: **0.261.125** on the foundation review branch, recorded in +`application/single_app/config.py`. This corrective commit does not independently +bump the application version; the parent integration owns the next version. + +Refs [#1509](https://github.com/microsoft/simplechat/issues/1509). + +## Issue and root cause + +M2/M4 integration exercised three boundaries that the initial result foundation +did not yet handle: + +| Boundary | Root cause | Correction | +| --- | --- | --- | +| A resumed native Analyze step retains its generic result. | Generic preparation called `prepare_analysis_attempt()` with the default `resume_from=None`, conflicting with the existing non-null native resume binding. | Reuse the active same-token guard without resetting request, source, or resume metadata. | +| A result commits, then the worker stops before its runtime checkpoint saves the digest. | Only digest-keyed commits existed; the runtime had no exact authenticated producer/input lookup. | Atomically commit an immutable producer/input receipt with the digest pointer and recover it without replaying producer work. | +| Gather uses web, URL, deep research, an agent/action, or permitted memory. | Source-free or document/upstream-only lineage could not truthfully describe these sources. | Explicit bounded external descriptors, server-catalog admission, and injected current reauthorization in an opt-in lineage version. | + +## Technical changes and scope + +`functions_orchestration_result_contracts.py` adds external-source descriptors +and receipt binding validation without changing public v1 task/reference fields. +`functions_orchestration_results.py` adds optional `input_fingerprint` and +`external_sources` persistence arguments, `recover_task_result()`, and external +access injection. `functions_workflow_result_store.py` reuses native lifecycle +guards and supports an immutable receipt plus digest commit in one existing +conditional transaction. + +Same producer/attempt/input and same digest is idempotent. A different digest +is an explicit `OrchestrationResultConflictError`, not a latest-result choice. +Only a missing exact receipt returns `None`; corrupt, deleted, unauthorized or +unavailable results fail. Readable payloads are fully verified during recovery. +Original attempt identities remain intact across later sibling failures; +cross-attempt consumption still needs an explicit authorized alias. + +External provenance contains original capability, opaque reference, audience, +and digest/revision information. It contains no credentials, fetchable URL +instructions, runtime callbacks or raw Fact Memory prompts. Current callbacks +are required for every external type, including public web capability checks. +Historical snapshots do not bypass current integration access or memory +audience restrictions. The durable bindings can be restored after restart +without treating an in-memory catalog as the source of truth. + +Unused extensions preserve the exact published v1 result bytes and existing +checkpoint fingerprints. Standalone/workflow APIs, runtime/adapters, routes, +artifact/rendering modules, and user-facing feature activation are not changed. + +See [the foundation API reference](../features/ORCHESTRATION_RENDERING_HARNESS.md) +for exact signatures, private manifest/lineage versions, limits and collision +semantics. + +## Validation + +`functional_tests/test_orchestration_result_recovery.py` uses the real native +work-unit lifecycle, including completed-unit reuse with `resume_from`, actual +owner tokens, cancellation, deletion and stale-worker rejection. It also covers +Cosmos/Blob restart recovery, receipt transaction rollback, competing writers, +corrupt receipts/data, original output ordering, partial output status, and +published-v1 byte fingerprints. + +`functional_tests/test_orchestration_external_results.py` covers all six external +source types, server-only alias admission, restored catalogs, strict descriptors, +current capability/resource/audience denial, revision/digest changes, +historical snapshots and inherited lineage. + +`functional_tests/test_orchestration_result_imports.py` executes real modules and +the new repair paths in fresh normal and optimized interpreters with network +blocked. Required operations and explicit subprocess checks still execute +under `-O`. Existing storage, native work recovery, source access, orchestration +checkpoint and exact-record JSON regressions remain part of the focused matrix. + +All validation uses external-I/O doubles; it does not claim live provider or +cloud throughput. These fixes recover retained data and provenance, not an +activated orchestration renderer or a completed epic. diff --git a/functional_tests/test_orchestration_external_results.py b/functional_tests/test_orchestration_external_results.py new file mode 100644 index 000000000..446098aec --- /dev/null +++ b/functional_tests/test_orchestration_external_results.py @@ -0,0 +1,273 @@ +# test_orchestration_external_results.py +""" +Bounded external Gather provenance, admission and current authorization. +Version: 0.261.125 +Implemented in: 0.261.125 + +No external resource is fetched. Server catalogs and current access metadata are +external-I/O doubles; real contracts, private storage and readers are exercised. +""" + +from copy import deepcopy +from dataclasses import replace +import json + +import pytest + +from functions_orchestration_result_contracts import ( + EXTERNAL_LINEAGE_VERSION, + EXTERNAL_SOURCE_VERSION, + RESULT_MANIFEST_VERSION, + ExternalSourceRef, + ResultContractError, +) +from functions_orchestration_results import ( + NamedOutput, + OrchestrationResultAccess, + OrchestrationResults, + ResultUnavailableError, +) +from test_support.orchestration_results import ResultFixture, complete +from test_support.versioning import assert_app_version_at_least + + +CAPABILITIES = { + "web": "web_search", "url": "url_fetch", "deep_research": "deep_research", + "agent": "agent_invoke", "action": "action_invoke", "fact_memory": "fact_memory", +} +PREPARED = { + "findings": ["Retained first finding.", "Retained final finding."], + "display_url": "https://public.example/retained-provenance", +} + + +def test_external_lineage_foundation_version(): + assert_app_version_at_least("0.261.125") + + +class ExternalFixture(ResultFixture): + def __init__(self, source_type="web"): + super().__init__(blob=True) + self.producer = replace( + self.producer, step_id="gather", capability_id="agent_invoke", contract_version="gather-v1", + ) + self.add_producer(self.producer) + reference = ExternalSourceRef( + source_type, CAPABILITIES[source_type], "server-resource-1", "personal:owner", + content_sha256="1" * 64, source_revision="revision-1", + ) + self.catalog_wire = {"admitted_source": reference.to_dict()} + self.current = {reference.identity(): reference} + self.enabled = {reference.capability_id} + self.audiences = {reference.audience} + self.calls = [] + self.service = self.reopen() + + def authorize_external(self, reference, *, producer, user_id, conversation_id): + self.calls.append((reference, producer, user_id, conversation_id)) + if ( + user_id != "owner" or conversation_id != "conversation-1" + or reference.capability_id not in self.enabled or reference.audience not in self.audiences + or reference.identity() not in self.current + ): + raise PermissionError("The current capability, resource or audience is unavailable.") + return self.current[reference.identity()] + + def reopen(self, *, catalog=True, authorizer=True): + self.catalog_wire = json.loads(json.dumps(self.catalog_wire)) + access = OrchestrationResultAccess( + user_id="owner", conversation_id="conversation-1", + read_conversation=lambda conversation_id: deepcopy(self.conversation), + read_run=lambda run_id: deepcopy(self.runs.get(run_id)), + source_resolver=self.resolve, source_metadata_reader=self.metadata, + external_source_catalog={ + alias: ExternalSourceRef.from_dict(value) for alias, value in self.catalog_wire.items() + } if catalog else {}, + external_source_authorizer=self.authorize_external if authorizer else None, + ) + return OrchestrationResults(self.restart().store, access) + + def retain(self, **changes): + options = { + "producer": self.producer, "role": "gather", "status": "complete", + "outputs": [NamedOutput("knowledge", "structured-v1", deepcopy(PREPARED), complete(1))], + "sources": [], "origin": "grounded", "guard_token": "owning-gather-token", + "external_sources": ("admitted_source",), + } + return self.service.persist_task_result(**{**options, **changes}) + + +@pytest.mark.parametrize("source_type", list(CAPABILITIES)) +def test_each_external_type_is_retained_and_reauthorized_after_restart(source_type): + fixture = ExternalFixture(source_type) + task = fixture.retain(input_fingerprint="a" * 64) + restarted = fixture.reopen(catalog=False) + recovered = restarted.recover_task_result(producer=fixture.producer, input_fingerprint="a" * 64) + reader = restarted.open_result(recovered.output("knowledge")) + value = reader.read_value() + metadata = reader.metadata() + assert recovered == task and value == PREPARED + assert metadata["origin"] == "grounded" and metadata["source_count"] == 0 + assert metadata["external_source_count"] == 1 + bindings = metadata["external_sources"] + assert bindings == [{"alias": "admitted_source", "reference": fixture.catalog_wire["admitted_source"]}] + assert bindings[0]["reference"]["version"] == EXTERNAL_SOURCE_VERSION + assert all(call[1:] == (fixture.producer, "owner", "conversation-1") for call in fixture.calls) + assert fixture.source_reads == [] + manifest = restarted.store.load_committed_orchestration_result( + "owner", "conversation-1", "run-1", "gather", task.output("knowledge").manifest_sha256, + ) + assert manifest["version"] == RESULT_MANIFEST_VERSION + assert manifest["lineage"]["version"] == EXTERNAL_LINEAGE_VERSION + assert manifest["lineage"]["external_sources"] == bindings + encoded = json.dumps(manifest) + assert "https://" not in encoded + assert "authorize_external" not in encoded and "owning-gather-token" not in encoded + + +@pytest.mark.parametrize("source_type", list(CAPABILITIES)) +@pytest.mark.parametrize("change", ["capability", "resource", "audience", "authorizer"]) +def test_current_external_access_is_required_even_for_public_or_snapshot_sources(source_type, change): + fixture = ExternalFixture(source_type) + task = fixture.retain(source_policy="snapshot", input_fingerprint="a" * 64) + if change == "capability": + fixture.enabled.clear() + elif change == "resource": + fixture.current.clear() + elif change == "audience": + fixture.audiences.clear() + restarted = fixture.reopen(authorizer=change != "authorizer") + with pytest.raises(PermissionError): + restarted.open_result(task.output("knowledge")) + with pytest.raises(PermissionError): + restarted.recover_task_result(producer=fixture.producer, input_fingerprint="a" * 64) + + +@pytest.mark.parametrize("field,value", [ + ("source_type", "action"), ("capability_id", "different-capability"), + ("reference_id", "different-resource"), ("audience", "group:foreign"), +]) +def test_current_authorizer_cannot_change_external_identity_or_audience(field, value): + fixture = ExternalFixture() + task = fixture.retain(source_policy="snapshot") + identity, reference = next(iter(fixture.current.items())) + fixture.current[identity] = replace(reference, **{field: value}) + with pytest.raises(ResultUnavailableError): + fixture.reopen().open_result(task.output("knowledge")) + + +@pytest.mark.parametrize("field,value", [("source_revision", "changed"), ("content_sha256", "2" * 64)]) +def test_current_policy_rejects_external_changes_but_snapshot_reads_original_data(field, value): + fixture = ExternalFixture() + current = fixture.retain() + historical = fixture.retain(source_policy="snapshot") + identity, reference = next(iter(fixture.current.items())) + fixture.current[identity] = replace(reference, **{field: value}) + with pytest.raises(ResultUnavailableError): + fixture.reopen().open_result(current.output("knowledge")) + reader = fixture.reopen().open_result(historical.output("knowledge")) + value = reader.read_value() + metadata = reader.metadata() + assert value == PREPARED and metadata["source_snapshot_changed"] is True + assert metadata["external_sources"][0]["reference"] == reference.to_dict() + with pytest.raises(ResultUnavailableError): + fixture.reopen().open_result(historical.output("knowledge"), require_current_sources=True) + + +def test_stored_admissions_can_be_restored_without_a_python_catalog_cache(): + fixture = ExternalFixture() + task = fixture.retain() + restarted = fixture.reopen(catalog=False) + reader = restarted.open_result(task.output("knowledge")) + restored = reader.metadata()["external_sources"] + with pytest.raises(ResultContractError): + restarted.access.admit_external_sources(("admitted_source",)) + fixture.catalog_wire = { + binding["alias"]: binding["reference"] for binding in json.loads(json.dumps(restored)) + } + fixture.service = fixture.reopen() + again = fixture.retain() + assert again == task + + +def test_transitive_external_lineage_is_authorized_using_the_original_producer(): + fixture = ExternalFixture("fact_memory") + task = fixture.retain(source_policy="snapshot") + child_producer = fixture.consumer() + child = fixture.service.persist_task_result( + producer=child_producer, role="reason", status="complete", + outputs=[NamedOutput("summary", "text-v1", "Prepared from retained facts.", complete(1))], + sources=[], origin="grounded", guard_token="child-token", + upstream=(task.output("knowledge"),), source_policy="snapshot", + ) + child_reader = fixture.reopen(catalog=False).open_result(child.output("summary")) + metadata = child_reader.metadata() + assert metadata["external_source_count"] == 1 + assert all(call[1] == fixture.producer for call in fixture.calls) + fixture.audiences.clear() + with pytest.raises(PermissionError): + child_reader.read_text() + + +@pytest.mark.parametrize("aliases", [ + ("not_admitted",), ("admitted_source", "admitted_source"), + "admitted_source", ({"url": "https://untrusted.example"},), +]) +def test_model_values_cannot_become_admitted_external_sources(aliases): + fixture = ExternalFixture() + with pytest.raises(ResultContractError): + fixture.retain(external_sources=aliases) + assert fixture.container.items == {} + + +def test_duplicate_external_identity_and_catalog_limits_are_explicit(): + fixture = ExternalFixture() + fixture.catalog_wire["duplicate"] = fixture.catalog_wire["admitted_source"] + fixture.service = fixture.reopen() + with pytest.raises(ResultContractError): + fixture.retain(external_sources=("admitted_source", "duplicate")) + fixture.catalog_wire = {f"alias_{index}": fixture.catalog_wire["admitted_source"] for index in range(65)} + with pytest.raises(ResultContractError): + fixture.reopen() + + +@pytest.mark.parametrize("extra", [ + {"url": "https://untrusted.example"}, {"credentials": "not-allowed"}, + {"callback": "not-allowed"}, {"prompt": "raw-memory-prompt"}, {"storage": "foreign"}, +]) +def test_external_wire_never_accepts_fetch_instructions_or_runtime_context(extra): + reference = ExternalSourceRef("web", "web_search", "server-ref", "personal:owner", source_revision="v1") + with pytest.raises(ResultContractError): + ExternalSourceRef.from_dict({**reference.to_dict(), **extra}) + + +@pytest.mark.parametrize("changes", [ + {"content_sha256": None, "source_revision": None}, + {"source_type": "untyped"}, + {"reference_id": "https://public.example/resource"}, + {"audience": "personal:owner\nforeign"}, + {"content_sha256": "not-a-digest"}, + {"source_revision": "v" * 257}, + {"source_revision": "raw\nprompt"}, + {"version": "unknown"}, +]) +def test_external_descriptor_is_strict_and_bounded(changes): + value = ExternalSourceRef("web", "web_search", "server-ref", "personal:owner", source_revision="v1").to_dict() + with pytest.raises(ResultContractError): + ExternalSourceRef.from_dict({**value, **changes}) + + +@pytest.mark.parametrize("returned", [None, True, {}, "authorized"]) +def test_external_authorizer_never_uses_success_shaped_fallbacks(returned): + fixture = ExternalFixture() + fixture.service.access.external_source_authorizer = lambda *args, **kwargs: returned + with pytest.raises(ResultUnavailableError): + fixture.retain() + assert fixture.container.items == {} + + +def test_external_lineage_cannot_be_mislabeled_as_source_free(): + fixture = ExternalFixture() + with pytest.raises(ResultContractError): + fixture.retain(origin="generated") + assert fixture.container.items == {} diff --git a/functional_tests/test_orchestration_result_imports.py b/functional_tests/test_orchestration_result_imports.py index 99eacd8d5..424db8ba5 100644 --- a/functional_tests/test_orchestration_result_imports.py +++ b/functional_tests/test_orchestration_result_imports.py @@ -52,9 +52,13 @@ def check(condition, message): with patch.object(socket.socket, "connect", no_network), patch.object(builtins, "__import__", guarded_import): for name in sys.argv[3:]: importlib.import_module(name) - from test_support.orchestration_results import ResultFixture, ROWS - from functions_orchestration_result_contracts import TaskResult - from functions_orchestration_results import OrchestrationResultAccess, ResultContractError + from dataclasses import replace + from functions_document_analysis_checkpoints import analysis_checkpoints_for_orchestration + from test_support.orchestration_results import ResultFixture, ROWS, complete, source + from functions_orchestration_result_contracts import ExternalSourceRef, TaskResult + from functions_orchestration_results import ( + NamedOutput, OrchestrationResultAccess, OrchestrationResults, ResultContractError, + ) fixture = ResultFixture(blob=True) saved = fixture.save(grounded=False) checkpoint = saved.to_dict() @@ -64,6 +68,60 @@ def check(condition, message): check(records == ROWS, "Complete result changed during cold restart") check(records[-1]["id"] == "last", "Final record was not read") check(bool(fixture.blobs.uploads), "Required persistence disappeared under optimized Python") + bound = fixture.save(grounded=False, input_fingerprint="a" * 64) + recovered = fixture.restart().recover_task_result( + producer=fixture.producer, input_fingerprint="a" * 64, + ) + check(recovered == bound, "Exact receipt recovery changed or disappeared") + external = ExternalSourceRef("web", "web_search", "server-ref", "personal:owner", source_revision="v1") + access = OrchestrationResultAccess( + user_id="owner", conversation_id="conversation-1", + read_conversation=fixture.service.access.read_conversation, + read_run=fixture.service.access.read_run, + external_source_catalog={"source": external}, + external_source_authorizer=lambda reference, **context: external, + ) + service = OrchestrationResults(fixture.service.store, access) + gathered = service.persist_task_result( + producer=fixture.producer, role="reason", status="complete", + outputs=[NamedOutput("prepared", "text-v1", "Retained external content.", complete(1))], + sources=[], origin="grounded", guard_token="server-attempt-token", + input_fingerprint="b" * 64, external_sources=("source",), + ) + access.external_source_catalog.clear() + retained = service.recover_task_result(producer=fixture.producer, input_fingerprint="b" * 64) + metadata = service.open_result(retained.output("prepared")).metadata() + check(retained == gathered and metadata["external_source_count"] == 1, "External lineage was only cached") + + native = ResultFixture() + original = native.producer + resumed_producer = replace(original, run_id="resumed-run", attempt_index=2) + native.add_producer(resumed_producer) + def checkpoints(producer, previous=None): + return analysis_checkpoints_for_orchestration( + producer.user_id, producer.conversation_id, producer.run_id, producer.step_id, + store=native.service.store, resume_run_id=previous, + authorize=lambda: native.service.access.authorize_producer(producer, for_write=True), + source_authorizer=lambda user_id, sources, require_snapshot: ( + native.service.access.authorize_sources(sources, require_snapshot=require_snapshot) + ), + ) + first = checkpoints(original) + first.prepare() + first.initialize({"operation": "analyze"}, [source()]) + resumed = checkpoints(resumed_producer, original.run_id) + resumed.prepare() + resumed.initialize({"operation": "analyze"}, [source()]) + result = native.service.persist_task_result( + producer=resumed_producer, role="reason", status="complete", + outputs=[NamedOutput("prepared", "text-v1", "Native retained data.", complete(1))], + sources=[source()], origin="grounded", guard_token=resumed.token, + input_fingerprint="c" * 64, + ) + guard = native.service.store._analysis_guard(resumed.binding, required=True) + check(guard["resume_from"] == first.binding, "Generic saving reset the native resume binding") + recovered = native.restart().recover_task_result(producer=resumed_producer, input_fingerprint="c" * 64) + check(recovered == result, "Native resumed receipt was not recovered") try: OrchestrationResultAccess( user_id="owner", conversation_id="conversation-1", diff --git a/functional_tests/test_orchestration_result_recovery.py b/functional_tests/test_orchestration_result_recovery.py new file mode 100644 index 000000000..6c5514602 --- /dev/null +++ b/functional_tests/test_orchestration_result_recovery.py @@ -0,0 +1,412 @@ +# test_orchestration_result_recovery.py +""" +Native resumed guards and exact committed-result recovery for orchestration. +Version: 0.261.125 +Implemented in: 0.261.125 + +Exercise real checkpoint/transport/facade APIs with external I/O doubled. +The parent integration owns the next application version increment. +""" + +from copy import deepcopy +from dataclasses import replace + +import pytest +from azure.cosmos.exceptions import CosmosBatchOperationError + +from content_screening.contracts import ScreeningError +from functions_document_analysis_checkpoints import analysis_checkpoints_for_orchestration +from functions_orchestration_result_contracts import ( + RESULT_MANIFEST_VERSION, + RESULT_RECEIPT_VERSION, + TASK_RESULT_VERSION, + InputBinding, + InputSpec, + ResultContractError, + canonical_bytes, + canonical_digest, +) +from functions_orchestration_results import NamedOutput, ResultUnavailableError +from functions_workflow_result_store import ( + AnalysisWorkUnitConflictError, + OrchestrationResultConflictError, + WorkflowResultIntegrityError, + WorkflowResultStorageUnavailableError, +) +from test_support.orchestration_results import COLUMNS, ROWS, ResultFixture, complete, source +from test_support.versioning import assert_app_version_at_least + + +def native_checkpoints(fixture, producer, *, previous=None): + return analysis_checkpoints_for_orchestration( + producer.user_id, producer.conversation_id, producer.run_id, producer.step_id, + store=fixture.service.store, resume_run_id=previous, + authorize=lambda: fixture.service.access.authorize_producer(producer, for_write=True), + source_authorizer=lambda user_id, sources, require_snapshot: ( + fixture.service.access.authorize_sources(sources, require_snapshot=require_snapshot) + ), + ) + + +INPUT_FINGERPRINT = "a" * 64 + + +def test_recovery_foundation_version(): + assert_app_version_at_least("0.261.125") + + +def persist_native_output(fixture, producer, checkpoints, **changes): + options = { + "producer": producer, "role": "reason", "status": "complete", + "outputs": [NamedOutput("findings", "records-v1", deepcopy(ROWS), complete(3), COLUMNS)], + "sources": [source()], "origin": "grounded", "guard_token": checkpoints.token, + } + return fixture.service.persist_task_result(**{**options, **changes}) + + +@pytest.mark.parametrize("blob", [False, True]) +def test_resumed_native_work_and_generic_results_share_the_owning_guard(blob): + fixture = ResultFixture(blob=blob) + first = native_checkpoints(fixture, fixture.producer) + request = {"operation": "analyze", "documents": ["document-1"]} + unit = {"work_unit_id": "window-1", "source": source(), "status": "completed"} + first.prepare() + first.initialize(request, [source()]) + first.source_loaded(source()) + claim = first.claim_unit(unit) + first.commit_unit(claim, unit, {"findings": ROWS}, "Retained native findings.") + first.cancel(reason="failed") + + resumed_producer = replace(fixture.producer, run_id="run-2", attempt_index=2) + fixture.add_producer(resumed_producer) + resumed = native_checkpoints(fixture, resumed_producer, previous="run-1") + resumed.prepare() + resumed.initialize(request, [source()]) + restored_unit = resumed.load_unit(unit) + assert restored_unit["candidate_result"]["findings"] == ROWS + before = fixture.service.store._analysis_guard(resumed.binding, required=True) + + task = persist_native_output(fixture, resumed_producer, resumed) + after = fixture.service.store._analysis_guard(resumed.binding, required=True) + for key in ("resume_from", "token", "request_digest", "request_registered", "source_snapshot_digest"): + assert after[key] == before[key] + assert after["resume_from"] == first.binding + rows = list(fixture.restart().open_result(task.output("findings")).iter_records()) + assert rows == ROWS + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, fixture.producer, first) + + competitor = native_checkpoints(fixture, resumed_producer, previous="run-1") + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, resumed_producer, competitor) + resumed.cancel() + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, resumed_producer, resumed) + retained = list(fixture.restart().open_result(task.output("findings")).iter_records()) + assert retained == ROWS + fixture.service.store.delete_orchestration_results("owner", "conversation-1", "run-2") + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, resumed_producer, resumed) + with pytest.raises(AnalysisWorkUnitConflictError): + fixture.restart().open_result(task.output("findings")) + + +@pytest.mark.parametrize("blob", [False, True]) +def test_crash_after_commit_recovers_exact_original_before_any_checkpoint(blob, monkeypatch): + fixture = ResultFixture(blob=blob) + producer = fixture.producer + missing = fixture.service.recover_task_result(producer=producer, input_fingerprint=INPUT_FINGERPRINT) + assert missing is None + checkpoints = native_checkpoints(fixture, producer) + checkpoints.prepare() + original_commit = fixture.service.store.commit_orchestration_result + + def commit_then_crash(*args, **kwargs): + original_commit(*args, **kwargs) + raise RuntimeError("Injected crash before the runtime captures the result descriptor.") + + with monkeypatch.context() as scoped: + scoped.setattr(fixture.service.store, "commit_orchestration_result", commit_then_crash) + with pytest.raises(RuntimeError, match="Injected crash"): + persist_native_output(fixture, producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + fixture.runs["run-1"]["status"] = "failed" + fixture.runs["run-1"]["latest_attempt_run_id"] = "retry-after-sibling-failure" + checkpoints.cancel(reason="failed") + restart = fixture.restart() + recovered = restart.recover_task_result(producer=producer, input_fingerprint=INPUT_FINGERPRINT) + recovered_again = fixture.restart().recover_task_result(producer=producer, input_fingerprint=INPUT_FINGERPRINT) + rows = list(restart.open_result(recovered.output("findings")).iter_records()) + assert recovered == recovered_again + assert rows == ROWS and rows[0]["id"] == "001" and rows[-1]["id"] == "last" + assert recovered.producer == producer + assert fixture.container.queries == [] + next_producer = fixture.consumer(run_id="run-2", attempt_index=2) + direct = InputSpec("data", InputBinding("analyze", "findings"), ("records-v1",)) + with pytest.raises(ResultContractError): + restart.resolve_input(direct, consumer=next_producer, task_results={"analyze": recovered}) + alias = InputSpec("data", InputBinding(existing_result="prior_result"), ("records-v1",)) + reader = restart.resolve_input( + alias, consumer=next_producer, task_results={}, + existing_results={"prior_result": recovered.output("findings")}, + ) + retained = list(reader.iter_records()) + assert retained == ROWS + + +def test_receipt_and_digest_commit_are_atomic_and_retry_does_not_pick_a_partial_commit(monkeypatch): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + checkpoints.prepare() + store = fixture.service.store + original_write = store._write_analysis_record + + def fail_companion(identity, record, **kwargs): + if record.get("key", "").startswith(RESULT_RECEIPT_VERSION): + fixture.container.fail_batch_at = 2 + try: + return original_write(identity, record, **kwargs) + finally: + fixture.container.fail_batch_at = None + + with monkeypatch.context() as scoped: + scoped.setattr(store, "_write_analysis_record", fail_companion) + with pytest.raises(CosmosBatchOperationError): + persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + final_rows = [row for row in fixture.container.items.values() if row.get("record_kind") == "final"] + missing = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert final_rows == [] and missing is None + saved = persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + recovered = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert recovered == saved + + +def test_same_input_receipt_is_immutable_not_latest_result_wins(): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + saved = persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + repeated = persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + assert repeated == saved + changed = [NamedOutput("findings", "records-v1", ROWS[:1], complete(1), COLUMNS)] + with pytest.raises(OrchestrationResultConflictError): + persist_native_output( + fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT, outputs=changed, + ) + original = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + other = persist_native_output( + fixture, fixture.producer, checkpoints, input_fingerprint="b" * 64, outputs=changed, + ) + exact_other = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint="b" * 64) + assert original == saved and exact_other == other and other != saved + + +def test_racing_receipt_writers_return_one_winner_not_an_ambiguous_catalog(monkeypatch): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + original_write = fixture.service.store._write_analysis_record + raced = [] + winner = [] + + def commit_competitor(): + winner.append(persist_native_output( + fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT, + )) + + def race(identity, record, **kwargs): + if record.get("key", "").startswith(RESULT_RECEIPT_VERSION) and not raced: + raced.append(True) + fixture.container.before_batch = commit_competitor + return original_write(identity, record, **kwargs) + + monkeypatch.setattr(fixture.service.store, "_write_analysis_record", race) + with pytest.raises(OrchestrationResultConflictError): + persist_native_output( + fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT, + outputs=[NamedOutput("findings", "records-v1", ROWS[:1], complete(1), COLUMNS)], + ) + recovered = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert recovered == winner[0] + receipts = [ + row for row in fixture.container.items.values() + if row.get("key", "").startswith(RESULT_RECEIPT_VERSION) + ] + assert len(receipts) == 1 + + +def test_unused_extensions_preserve_published_v1_bytes_and_optional_fingerprints(): + default_fixture = ResultFixture() + default = default_fixture.save() + explicit = ResultFixture().save(input_fingerprint=None, external_sources=()) + reference = default.output("findings") + manifest = default_fixture.service.store.load_committed_orchestration_result( + "owner", "conversation-1", "run-1", "analyze", reference.manifest_sha256, + ) + # Captured from the complete published facade in commit 18fc08ac, not a + # reimplementation of its serializer or a value tied to the app version. + assert reference.manifest_sha256 == "a1a6021aa32908a7c4657ebeb8c4a005a379cddbc809dd990710f110c8948fd6" + assert canonical_digest(default.to_dict()) == "42bbe2b028fa4f3298f980b18398532fb3b535551ae60ce23821a8118221d9ce" + assert len(canonical_bytes(default.to_dict())) == 1281 + assert explicit == default and manifest["version"] == TASK_RESULT_VERSION + assert "input_fingerprint" not in manifest and "version" not in manifest["lineage"] + missing = default_fixture.service.recover_task_result( + producer=default_fixture.producer, input_fingerprint=INPUT_FINGERPRINT, + ) + assert missing is None + + +@pytest.mark.parametrize("status", ["pending", "invalid", "failed", "cancelled", "unavailable"]) +def test_pending_jobs_and_unusable_results_never_create_terminal_receipts(status): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + with pytest.raises(ResultContractError): + persist_native_output( + fixture, fixture.producer, checkpoints, status=status, input_fingerprint=INPUT_FINGERPRINT, + outputs=[NamedOutput("findings", "records-v1", [], complete(0, status=status, expected=3), COLUMNS)], + ) + assert fixture.container.items == {} + + +def test_recovery_retains_partial_state_and_requires_explicit_partial_read(): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + saved = persist_native_output( + fixture, fixture.producer, checkpoints, status="partial", input_fingerprint=INPUT_FINGERPRINT, + outputs=[NamedOutput("findings", "records-v1", ROWS[:1], complete(1, status="partial", expected=3), COLUMNS)], + ) + recovered = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert recovered == saved and recovered.status == "partial" + with pytest.raises(ResultContractError): + fixture.service.open_result(recovered.output("findings")) + reader = fixture.service.open_result(recovered.output("findings"), allow_partial=True) + rows = list(reader.iter_records()) + assert rows == ROWS[:1] and reader.completeness.limitations + + +def test_partial_task_receipt_preserves_unreadable_failure_output_without_promoting_it(): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + saved = persist_native_output( + fixture, fixture.producer, checkpoints, status="partial", input_fingerprint=INPUT_FINGERPRINT, + outputs=[ + NamedOutput("retained", "records-v1", ROWS, complete(3), COLUMNS), + NamedOutput("failed", "records-v1", [], complete(0, status="failed", expected=1), COLUMNS), + ], + ) + recovered = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert recovered == saved and recovered.status == "partial" + with pytest.raises(ResultContractError): + fixture.service.open_result(recovered.output("failed"), allow_partial=True) + + +@pytest.mark.parametrize("change", ["source_deleted", "screening", "revision", "owner", "scope", "attempt", "cleanup"]) +def test_receipt_recovery_reauthorizes_current_source_and_producer_state(change): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + if change == "source_deleted": + fixture.sources.clear() + elif change == "screening": + fixture.held.add("document-1") + elif change == "revision": + fixture.sources["document-1"]["source_revision"] = "new-revision" + elif change == "owner": + fixture.conversation["user_id"] = "foreign" + elif change == "scope": + fixture.sources["document-1"].update(scope="group", scope_id="foreign-group") + elif change == "attempt": + fixture.runs["run-1"]["attempt_index"] = 2 + else: + fixture.service.store.delete_orchestration_results("owner", "conversation-1", "run-1") + with pytest.raises((PermissionError, ScreeningError, AnalysisWorkUnitConflictError)): + fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + + +@pytest.mark.parametrize("change", [ + "fingerprint", "producer", "coerced_attempt", "reference", "missing_guard", "missing_commit", +]) +def test_corrupt_or_incomplete_receipt_is_not_reported_as_missing(change): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + rows = fixture.container.items + receipt = next(row for row in rows.values() if row.get("key", "").startswith(RESULT_RECEIPT_VERSION)) + if change == "fingerprint": + receipt["binding"]["input_fingerprint"] = "b" * 64 + elif change == "producer": + receipt["binding"]["producer"]["user_id"] = "foreign" + elif change == "coerced_attempt": + receipt["binding"]["producer"]["attempt_index"] = True + elif change == "reference": + receipt["reference"]["size_bytes"] += 1 + else: + kind = "lifecycle" if change == "missing_guard" else "final" + key = next( + key for key, row in rows.items() + if row.get("record_kind") == kind and not row.get("key", "").startswith(RESULT_RECEIPT_VERSION) + ) + del rows[key] + with pytest.raises((WorkflowResultIntegrityError, AnalysisWorkUnitConflictError)): + fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + + +def test_receipt_preserves_named_output_order_and_rejects_foreign_lookup_before_storage(): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + saved = persist_native_output( + fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT, + outputs=[ + NamedOutput("z_text", "text-v1", "First named output.", complete(1)), + NamedOutput("a_data", "structured-v1", {"last": True}, complete(1)), + ], + ) + recovered = fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert recovered == saved + assert tuple(reference.output_name for reference in recovered.outputs) == ("z_text", "a_data") + fixture.container.fail_reads = True + with pytest.raises(ResultUnavailableError): + fixture.service.recover_task_result( + producer=replace(fixture.producer, user_id="foreign"), input_fingerprint=INPUT_FINGERPRINT, + ) + + +def test_receipt_recovery_exhausts_data_integrity_checks_and_never_falls_back(): + fixture = ResultFixture(blob=True) + checkpoints = native_checkpoints(fixture, fixture.producer) + task = persist_native_output(fixture, fixture.producer, checkpoints) + store = fixture.service.store + args = ("owner", "conversation-1", "run-1", "analyze") + manifest = store.load_committed_orchestration_result(*args, task.output("findings").manifest_sha256) + manifest.update( + version=RESULT_MANIFEST_VERSION, input_fingerprint=INPUT_FINGERPRINT, output_order=list(manifest["outputs"]), + ) + manifest["outputs"]["findings"]["content_sha256"] = "0" * 64 + stored = store.save_orchestration(*args, manifest, guard_token=checkpoints.token, require_analysis_guard=True) + store.commit_orchestration_result( + *args, stored, guard_token=checkpoints.token, producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT, + ) + with pytest.raises(ResultContractError) as error: + fixture.restart().recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + assert error.value.code == "result_content_integrity" + with pytest.raises(WorkflowResultStorageUnavailableError): + fixture.restart(blobs=False).recover_task_result(producer=fixture.producer, input_fingerprint=INPUT_FINGERPRINT) + + +@pytest.mark.parametrize("stop", ["cancel", "delete"]) +def test_receipt_commit_loses_native_cas_when_cancellation_or_deletion_wins(stop, monkeypatch): + fixture = ResultFixture() + checkpoints = native_checkpoints(fixture, fixture.producer) + original_write = fixture.service.store._write_analysis_record + + def race_stop(identity, record, **kwargs): + if record.get("key", "").startswith(RESULT_RECEIPT_VERSION): + fixture.container.before_batch = ( + checkpoints.cancel if stop == "cancel" + else lambda: fixture.service.store.fence_analysis_attempt(identity) + ) + return original_write(identity, record, **kwargs) + + monkeypatch.setattr(fixture.service.store, "_write_analysis_record", race_stop) + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT) + assert not any(row.get("record_kind") == "final" for row in fixture.container.items.values()) + with pytest.raises(AnalysisWorkUnitConflictError): + persist_native_output(fixture, fixture.producer, checkpoints, input_fingerprint=INPUT_FINGERPRINT)