From ca5b7180e0d0458cf38592f05273475019dfb99c Mon Sep 17 00:00:00 2001 From: alvaro Date: Sat, 8 Aug 2026 13:31:04 +0200 Subject: [PATCH 01/14] initial recipe_language and test --- src/pypts/recipe_language.py | 364 +++++++++++++++++++++++ tests/unit_tests/test_recipe_language.py | 111 +++++++ 2 files changed, 475 insertions(+) create mode 100644 src/pypts/recipe_language.py create mode 100644 tests/unit_tests/test_recipe_language.py diff --git a/src/pypts/recipe_language.py b/src/pypts/recipe_language.py new file mode 100644 index 0000000..177d121 --- /dev/null +++ b/src/pypts/recipe_language.py @@ -0,0 +1,364 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later +"""The framework-independent contract for the recipe YAML language. + +This module deliberately accepts already-loaded Python dictionaries. Loading +YAML, retaining source locations, and constructing runtime steps are separate +concerns which will be introduced by the parser and integration work. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping + + +CANONICAL_RECIPE_VERSION = "1.0.0" + + +@dataclass(frozen=True) +class Diagnostic: + """A language-contract finding for an already-loaded recipe document.""" + + code: str + message: str + path: tuple[str | int, ...] = () + severity: str = "error" + + +@dataclass(frozen=True) +class ValidationResult: + diagnostics: tuple[Diagnostic, ...] = () + + @property + def errors(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "error") + + @property + def warnings(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "warning") + + @property + def is_valid(self) -> bool: + return not self.errors + + +@dataclass(frozen=True) +class FieldSpec: + name: str + value_type: type | tuple[type, ...] | None = None + required: bool = False + description: str = "" + + +@dataclass(frozen=True) +class StepSpec: + """Declarative contract for one registered recipe step type.""" + + name: str + fields: tuple[FieldSpec, ...] = () + required_inputs: tuple[str, ...] = () + example: Mapping[str, Any] = field(default_factory=dict) + description: str = "" + source_allowed: bool = True + + @property + def fields_by_name(self) -> dict[str, FieldSpec]: + return {field.name: field for field in self.fields} + + +COMMON_STEP_FIELDS = ( + FieldSpec("steptype", str, required=True, description="Registered step type."), + FieldSpec("step_name", str, required=True, description="Human-readable step name."), + FieldSpec("description", str, required=True, description="Purpose of the step."), + FieldSpec("id", str, description="Optional stable step identifier."), + FieldSpec("skip", bool, description="Skip execution; defaults to false."), + FieldSpec("critical", bool, description="Stop on error when policy permits continuation."), + FieldSpec("continue_on_error", bool, description="Per-step error policy."), + FieldSpec("input_mapping", dict, description="Named input sources."), + FieldSpec("output_mapping", dict, description="Named verdicts and destinations."), +) + + +def _step_spec( + name: str, + *fields: FieldSpec, + required_inputs: tuple[str, ...] = (), + example: Mapping[str, Any], + description: str, + source_allowed: bool = True, +) -> StepSpec: + return StepSpec(name, COMMON_STEP_FIELDS + fields, required_inputs, example, description, source_allowed) + + +STEP_SPECS = ( + _step_spec( + "PythonModuleStep", + FieldSpec("action_type", str, required=True), + FieldSpec("module", str, required=True), + FieldSpec("method_name", str), + example={"steptype": "PythonModuleStep", "step_name": "Run test", "description": "Run a Python test method.", "action_type": "method", "module": "tests.py", "method_name": "run", "input_mapping": {}, "output_mapping": {}}, + description="Calls a method or reads/writes an attribute in a Python module.", + ), + _step_spec( + "SequenceStep", + FieldSpec("sequence", dict, required=True), + example={"steptype": "SequenceStep", "step_name": "Run calibration", "description": "Run an internal sequence.", "sequence": {"type": "internal", "name": "Calibration"}, "input_mapping": {}, "output_mapping": {}}, + description="Runs another sequence as a step.", + ), + _step_spec( + "UserInteractionStep", + example={"steptype": "UserInteractionStep", "step_name": "Confirm", "description": "Ask the operator to confirm.", "input_mapping": {"message": {"type": "direct", "value": "Continue?"}}, "output_mapping": {"output": {"type": "passfail"}}}, + description="Displays an operator interaction prompt.", + ), + _step_spec( + "WaitStep", + required_inputs=("wait_time",), + example={"steptype": "WaitStep", "step_name": "Stabilize", "description": "Wait for hardware stabilization.", "input_mapping": {"wait_time": {"type": "direct", "value": 1}}, "output_mapping": {}}, + description="Waits for a non-negative duration in seconds.", + ), + _step_spec( + "UserLoadingStep", + FieldSpec("file_save_location", dict), + example={"steptype": "UserLoadingStep", "step_name": "Load configuration", "description": "Ask the operator for a file.", "input_mapping": {"message": {"type": "direct", "value": "Choose a file"}}, "output_mapping": {"output": {"type": "passfail"}}}, + description="Prompts the operator to select a file.", + ), + _step_spec( + "UserRunMethodStep", + FieldSpec("trigger_response", (str, list, dict)), + FieldSpec("action_type", str), + FieldSpec("module", str), + FieldSpec("method_name", str), + example={"steptype": "UserRunMethodStep", "step_name": "Run calibration", "description": "Run on operator confirmation.", "trigger_response": "run", "action_type": "method", "module": "tests.py", "method_name": "calibrate", "input_mapping": {}, "output_mapping": {"output": {"type": "passfail"}}}, + description="Optionally runs a Python method after an operator response.", + ), + _step_spec( + "UserWriteStep", + example={"steptype": "UserWriteStep", "step_name": "Enter value", "description": "Ask the operator for a value.", "input_mapping": {"message": {"type": "direct", "value": "Enter value"}}, "output_mapping": {"output": {"type": "local", "local_name": "value"}}}, + description="Writes an operator-provided value to a configured destination.", + ), + _step_spec( + "SerialNumberStep", + example={"steptype": "SerialNumberStep", "step_name": "Scan serial number", "description": "Capture the device serial number.", "input_mapping": {}, "output_mapping": {}}, + description="Captures the device serial number.", + ), + _step_spec( + "SSHConnectStep", + example={"steptype": "SSHConnectStep", "step_name": "Connect", "description": "Open the SSH connection."}, + description="Opens the SSH client stored in recipe globals.", + ), + _step_spec( + "SSHCloseStep", + example={"steptype": "SSHCloseStep", "step_name": "Disconnect", "description": "Close the SSH connection."}, + description="Closes the SSH client stored in recipe globals.", + ), + _step_spec( + "SSHUploadStep", + FieldSpec("files", list, required=True), + FieldSpec("permissions", (int, str)), + FieldSpec("skip_if_sha256_match", bool), + FieldSpec("local_package", str), + example={"steptype": "SSHUploadStep", "step_name": "Deploy", "description": "Upload a file to the target.", "files": [{"local": "bin/tool", "remote": "/tmp/tool"}], "output_mapping": {"passed": {"type": "passfail"}}}, + description="Uploads files through an SSH connection.", + ), + _step_spec( + "IndexedStep", + example={"steptype": "IndexedStep", "step_name": "Indexed operation", "description": "Reserved runtime wrapper step.", "input_mapping": {}, "output_mapping": {}}, + description="Reserved for the runtime's automatic indexed-step wrapper.", + source_allowed=False, + ), +) + +STEP_SPECS_BY_NAME = {spec.name.casefold(): spec for spec in STEP_SPECS} + +HEADER_FIELDS = ( + FieldSpec("name", str, required=True), FieldSpec("version", str, required=True), + FieldSpec("recipe_version", str, required=True), FieldSpec("description", str, required=True), + FieldSpec("main_sequence", str, required=True), FieldSpec("globals", dict, required=True), + FieldSpec("continue_on_error", bool), FieldSpec("report", str), + FieldSpec("report_name_include_serial", bool), FieldSpec("test_package", str), +) +SEQUENCE_FIELDS = ( + FieldSpec("sequence_name", str, required=True), FieldSpec("description", str, required=True), + FieldSpec("parameters", dict, required=True), FieldSpec("outputs", dict, required=True), + FieldSpec("locals", dict, required=True), FieldSpec("setup_steps", list, required=True), + FieldSpec("steps", list, required=True), FieldSpec("teardown_steps", list, required=True), + FieldSpec("serial_number", (str, int), description="Legacy runtime-ignored sequence metadata."), +) + + +def canonical_step_type(step_type: str) -> str | None: + spec = STEP_SPECS_BY_NAME.get(step_type.casefold()) if isinstance(step_type, str) else None + return spec.name if spec else None + + +def _type_name(value_type: type | tuple[type, ...]) -> str: + values = value_type if isinstance(value_type, tuple) else (value_type,) + return " or ".join(value.__name__ for value in values) + + +def _check_fields(value: Mapping[str, Any], fields: Iterable[FieldSpec], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: + specs = {spec.name: spec for spec in fields} + for name, spec in specs.items(): + if spec.required and name not in value: + diagnostics.append(Diagnostic("missing-field", f"Missing required field '{name}'.", path + (name,))) + elif name in value and spec.value_type is not None and (type(value[name]) is not bool if spec.value_type is bool else not isinstance(value[name], spec.value_type)): + diagnostics.append(Diagnostic("invalid-field-type", f"Field '{name}' must be {_type_name(spec.value_type)}.", path + (name,))) + for name in value: + if name not in specs: + diagnostics.append(Diagnostic("unknown-field", f"Unknown field '{name}'.", path + (name,))) + + +def _validate_input_mappings(mapping: Mapping[str, Any], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: + indexed_lengths: list[int] = [] + for name, config in mapping.items(): + item_path = path + (name,) + if not isinstance(config, Mapping): + diagnostics.append(Diagnostic("invalid-input-mapping", "Input mapping must be a dictionary.", item_path)) + continue + source = config.get("type", "direct") + if source not in {"direct", "local", "global", "method"}: + diagnostics.append(Diagnostic("unknown-input-source", f"Unknown input source '{source}'.", item_path + ("type",))) + continue + required_key = {"direct": "value", "local": "local_name", "global": "global_name", "method": "value"}[source] + if required_key not in config: + diagnostics.append(Diagnostic("missing-input-source-value", f"Input source '{source}' requires '{required_key}'.", item_path)) + if "indexed" in config and type(config["indexed"]) is not bool: + diagnostics.append(Diagnostic("invalid-indexed-flag", "'indexed' must be boolean.", item_path + ("indexed",))) + if config.get("indexed"): + if source != "direct" or not isinstance(config.get("value"), list): + diagnostics.append(Diagnostic("invalid-indexed-input", "Indexed inputs must be direct lists.", item_path)) + else: + indexed_lengths.append(len(config["value"])) + if len(set(indexed_lengths)) > 1: + diagnostics.append(Diagnostic("unequal-indexed-inputs", "Indexed input lists must have equal lengths.", path)) + + +def _validate_output_mappings(mapping: Mapping[str, Any], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: + verdicts: list[str] = [] + requirements = {"equals": "value", "range": ("min", "max"), "local": "local_name", "global": "global_name"} + for name, config in mapping.items(): + item_path = path + (name,) + if not isinstance(config, Mapping) or not isinstance(config.get("type"), str): + diagnostics.append(Diagnostic("invalid-output-mapping", "Output mapping requires a string 'type'.", item_path)) + continue + kind = config["type"] + if kind not in {"passfail", "equals", "range", "passthrough", "local", "global", "image"}: + diagnostics.append(Diagnostic("unknown-output-type", f"Unknown output type '{kind}'.", item_path + ("type",))) + continue + if kind in {"passfail", "equals", "range", "passthrough"}: + verdicts.append(kind) + required = requirements.get(kind, ()) + required = (required,) if isinstance(required, str) else required + for field_name in required: + if field_name not in config: + diagnostics.append(Diagnostic("missing-output-field", f"Output type '{kind}' requires '{field_name}'.", item_path)) + if "passthrough" in verdicts and len(verdicts) != 1: + diagnostics.append(Diagnostic("mixed-passthrough", "'passthrough' must be the sole verdict mapping.", path)) + + +def _validate_step(step: Any, path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> str | None: + if not isinstance(step, Mapping): + diagnostics.append(Diagnostic("invalid-step", "Step must be a dictionary.", path)) + return None + step_type = step.get("steptype") + canonical_name = canonical_step_type(step_type) + if canonical_name is None: + diagnostics.append(Diagnostic("unknown-step-type", f"Unknown step type '{step_type}'.", path + ("steptype",))) + return None + spec = STEP_SPECS_BY_NAME[canonical_name.casefold()] + if not spec.source_allowed: + diagnostics.append(Diagnostic("internal-step-type", f"{canonical_name} is created by the runtime and cannot be written in a recipe.", path + ("steptype",))) + return canonical_name + _check_fields(step, spec.fields, path, diagnostics) + if "input_mapping" in step and isinstance(step["input_mapping"], Mapping): + _validate_input_mappings(step["input_mapping"], path + ("input_mapping",), diagnostics) + for required in spec.required_inputs: + if required not in step["input_mapping"]: + diagnostics.append(Diagnostic("missing-required-input", f"{canonical_name} requires input '{required}'.", path + ("input_mapping", required))) + elif spec.required_inputs: + diagnostics.append(Diagnostic("missing-input-mapping", f"{canonical_name} requires an input_mapping.", path + ("input_mapping",))) + if "output_mapping" in step and isinstance(step["output_mapping"], Mapping): + _validate_output_mappings(step["output_mapping"], path + ("output_mapping",), diagnostics) + if canonical_name == "PythonModuleStep": + if step.get("action_type") not in {"method", "read_attribute", "write_attribute"}: + diagnostics.append(Diagnostic("invalid-action-type", "PythonModuleStep action_type must be method, read_attribute, or write_attribute.", path + ("action_type",))) + if step.get("action_type") == "method" and not step.get("method_name"): + diagnostics.append(Diagnostic("missing-method-name", "PythonModuleStep method action requires method_name.", path + ("method_name",))) + if canonical_name == "SequenceStep": + sequence = step.get("sequence") + if not isinstance(sequence, Mapping) or sequence.get("type") != "internal" or not isinstance(sequence.get("name"), str): + diagnostics.append(Diagnostic("invalid-sequence-reference", "SequenceStep requires sequence.type 'internal' and string sequence.name.", path + ("sequence",))) + if canonical_name == "UserLoadingStep" and "file_save_location" in step: + location = step["file_save_location"] + if not isinstance(location, Mapping) or location.get("type") not in {"local", "global"} or not isinstance(location.get("variable"), str): + diagnostics.append(Diagnostic("invalid-file-save-location", "file_save_location requires type local/global and string variable.", path + ("file_save_location",))) + return canonical_name + + +def validate_recipe_documents(documents: Iterable[Any]) -> ValidationResult: + """Validate already-loaded documents against the canonical language model. + + The function has no YAML dependency and intentionally performs no runtime + imports or execution. A future parser will supply source spans and YAML + loading before calling this contract validator. + """ + docs = list(documents) + diagnostics: list[Diagnostic] = [] + if not docs: + return ValidationResult((Diagnostic("empty-recipe", "A recipe requires a header and at least one sequence."),)) + header = docs[0] + if not isinstance(header, Mapping): + return ValidationResult((Diagnostic("invalid-header", "The first document must be the recipe header.", (0,)),)) + _check_fields(header, HEADER_FIELDS, (0,), diagnostics) + sequences: dict[str, Mapping[str, Any]] = {} + sequence_step_types: dict[str, dict[str, list[str]]] = {} + for doc_index, sequence in enumerate(docs[1:], start=1): + path = (doc_index,) + if not isinstance(sequence, Mapping): + diagnostics.append(Diagnostic("invalid-sequence", "Sequence document must be a dictionary.", path)) + continue + _check_fields(sequence, SEQUENCE_FIELDS, path, diagnostics) + name = sequence.get("sequence_name") + if isinstance(name, str): + if name in sequences: + diagnostics.append(Diagnostic("duplicate-sequence", f"Duplicate sequence '{name}'.", path + ("sequence_name",))) + sequences[name] = sequence + if "serial_number" in sequence: + diagnostics.append(Diagnostic("legacy-sequence-field", "'serial_number' is runtime-ignored legacy metadata.", path + ("serial_number",), "warning")) + sections: dict[str, list[str]] = {} + for section in ("setup_steps", "steps", "teardown_steps"): + values = sequence.get(section, []) + section_types: list[str] = [] + if isinstance(values, list): + for index, step in enumerate(values): + step_type = _validate_step(step, path + (section, index), diagnostics) + if step_type: + section_types.append(step_type) + sections[section] = section_types + if isinstance(name, str): + sequence_step_types[name] = sections + if isinstance(header, Mapping) and isinstance(header.get("main_sequence"), str) and header["main_sequence"] not in sequences: + diagnostics.append(Diagnostic("unknown-main-sequence", f"Main sequence '{header['main_sequence']}' does not exist.", (0, "main_sequence"))) + for sequence_name, sequence in sequences.items(): + for section in ("setup_steps", "steps", "teardown_steps"): + for index, step in enumerate(sequence.get(section, [])): + if isinstance(step, Mapping) and canonical_step_type(step.get("steptype")) == "SequenceStep": + target = step.get("sequence", {}).get("name") if isinstance(step.get("sequence"), Mapping) else None + if isinstance(target, str) and target not in sequences: + diagnostics.append(Diagnostic("unknown-sequence-reference", f"Sequence '{sequence_name}' references unknown sequence '{target}'.", (sequence_name, section, index, "sequence", "name"))) + kinds = sequence_step_types.get(sequence_name, {}) + all_types = [kind for values in kinds.values() for kind in values] + if any(kind.startswith("SSH") for kind in all_types): + globals_data = header.get("globals", {}) if isinstance(header, Mapping) else {} + for required in ("ssh_client", "host", "user", "port"): + if required not in globals_data: + diagnostics.append(Diagnostic("missing-ssh-global", f"SSH step requires global '{required}'.", (0, "globals", required))) + if "password" not in globals_data and "private_key" not in globals_data: + diagnostics.append(Diagnostic("missing-ssh-credential", "SSH steps require password or private_key global.", (0, "globals"))) + if "SSHUploadStep" in kinds.get("steps", []) and "SSHConnectStep" not in kinds.get("setup_steps", []): + diagnostics.append(Diagnostic("missing-ssh-connect", f"Sequence '{sequence_name}' uses SSHUploadStep without setup SSHConnectStep.", (sequence_name, "steps"))) + if "SSHConnectStep" in kinds.get("setup_steps", []) and "SSHCloseStep" not in kinds.get("teardown_steps", []): + diagnostics.append(Diagnostic("missing-ssh-close", f"Sequence '{sequence_name}' setup SSHConnectStep requires teardown SSHCloseStep.", (sequence_name, "teardown_steps"))) + return ValidationResult(tuple(diagnostics)) diff --git a/tests/unit_tests/test_recipe_language.py b/tests/unit_tests/test_recipe_language.py new file mode 100644 index 0000000..1848219 --- /dev/null +++ b/tests/unit_tests/test_recipe_language.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later + +from pathlib import Path + +import pytest +import yaml + +from pypts.recipe_language import STEP_SPECS, canonical_step_type, validate_recipe_documents + + +RECIPES = Path(__file__).parents[2] / "src" / "pypts" / "recipes" + + +def _recipe_with_step(step, *, contextual=True): + sequence = { + "sequence_name": "Main", + "description": "Main sequence.", + "parameters": {}, "outputs": {}, "locals": {}, + "setup_steps": [], "steps": [step], "teardown_steps": [], + } + step_type = canonical_step_type(step["steptype"]) + documents = [ + { + "name": "Language contract", + "version": "1.0", + "recipe_version": "1.0.0", + "description": "Contract fixture.", + "main_sequence": "Main", + "globals": {"ssh_client": None, "host": "target", "user": "root", "port": 22, "password": "secret"}, + }, + ] + if contextual and step_type == "SequenceStep": + documents.append({ + "sequence_name": step["sequence"]["name"], "description": "Target sequence.", + "parameters": {}, "outputs": {}, "locals": {}, + "setup_steps": [], "steps": [], "teardown_steps": [], + }) + if contextual and step_type == "SSHUploadStep": + sequence["setup_steps"] = [{ + "steptype": "SSHConnectStep", "step_name": "Connect", "description": "Connect.", + }] + sequence["teardown_steps"] = [{ + "steptype": "SSHCloseStep", "step_name": "Close", "description": "Close.", + }] + documents.append(sequence) + return documents + + +@pytest.mark.parametrize("path", sorted(path for path in RECIPES.glob("*.yml") if path.name != "subsequence_executions_draft.yml")) +def test_working_recipe_corpus_conforms(path): + result = validate_recipe_documents(yaml.safe_load_all(path.read_text(encoding="utf-8"))) + assert result.is_valid, result.errors + + +def test_empty_draft_is_not_a_recipe(): + draft = RECIPES / "subsequence_executions_draft.yml" + result = validate_recipe_documents(yaml.safe_load_all(draft.read_text(encoding="utf-8"))) + assert [item.code for item in result.errors] == ["empty-recipe"] + + +@pytest.mark.parametrize("spec", [spec for spec in STEP_SPECS if spec.source_allowed], ids=lambda spec: spec.name) +def test_every_step_spec_has_a_valid_canonical_example(spec): + result = validate_recipe_documents(_recipe_with_step(dict(spec.example))) + assert result.is_valid, result.errors + + +def test_step_types_are_case_insensitive_but_have_one_canonical_name(): + assert canonical_step_type("pythonmodulestep") == "PythonModuleStep" + assert canonical_step_type("UnknownStep") is None + + +def test_internal_indexed_step_cannot_be_written_in_a_recipe(): + step = { + "steptype": "IndexedStep", "step_name": "Internal", "description": "Internal wrapper.", + "input_mapping": {}, "output_mapping": {}, + } + codes = {item.code for item in validate_recipe_documents(_recipe_with_step(step)).errors} + assert "internal-step-type" in codes + + +def test_implicit_direct_input_is_accepted_as_legacy_syntax(): + step = { + "steptype": "WaitStep", "step_name": "Wait", "description": "Wait.", + "input_mapping": {"wait_time": {"value": 1}}, "output_mapping": {}, + } + assert validate_recipe_documents(_recipe_with_step(step)).is_valid + + +def test_mapping_and_sequence_contract_failures_are_reported(): + step = { + "steptype": "SequenceStep", "step_name": "Missing", "description": "Missing target.", + "sequence": {"type": "internal", "name": "NoSuchSequence"}, + "input_mapping": { + "left": {"type": "direct", "value": [1], "indexed": True}, + "right": {"type": "direct", "value": [1, 2], "indexed": True}, + }, + "output_mapping": {"result": {"type": "passthrough"}, "passed": {"type": "passfail"}}, + } + codes = {item.code for item in validate_recipe_documents(_recipe_with_step(step, contextual=False)).errors} + assert {"unequal-indexed-inputs", "mixed-passthrough", "unknown-sequence-reference"} <= codes + + +def test_ssh_lifecycle_is_part_of_the_contract(): + step = { + "steptype": "SSHUploadStep", "step_name": "Upload", "description": "Upload.", + "files": [], "output_mapping": {}, + } + result = validate_recipe_documents(_recipe_with_step(step, contextual=False)) + assert "missing-ssh-connect" in {item.code for item in result.errors} From 1289e5e6135b6b92517b132a604f1968b1e8569b Mon Sep 17 00:00:00 2001 From: alvaro Date: Sat, 8 Aug 2026 15:09:32 +0200 Subject: [PATCH 02/14] first implementation of parser --- src/pypts/recipe_language.py | 41 ++ src/pypts/recipe_parser.py | 617 +++++++++++++++++++++++++ tests/unit_tests/test_recipe_parser.py | 207 +++++++++ 3 files changed, 865 insertions(+) create mode 100644 src/pypts/recipe_parser.py create mode 100644 tests/unit_tests/test_recipe_parser.py diff --git a/src/pypts/recipe_language.py b/src/pypts/recipe_language.py index 177d121..acc58f9 100644 --- a/src/pypts/recipe_language.py +++ b/src/pypts/recipe_language.py @@ -17,6 +17,23 @@ CANONICAL_RECIPE_VERSION = "1.0.0" +@dataclass(frozen=True) +class SourcePosition: + """A one-based source position with a zero-based character offset.""" + + line: int + column: int + offset: int + + +@dataclass(frozen=True) +class SourceSpan: + """Half-open source range.""" + + start: SourcePosition + end: SourcePosition + + @dataclass(frozen=True) class Diagnostic: """A language-contract finding for an already-loaded recipe document.""" @@ -25,6 +42,8 @@ class Diagnostic: message: str path: tuple[str | int, ...] = () severity: str = "error" + source_name: str | None = None + span: SourceSpan | None = None @dataclass(frozen=True) @@ -221,6 +240,15 @@ def _validate_input_mappings(mapping: Mapping[str, Any], path: tuple[str | int, if source not in {"direct", "local", "global", "method"}: diagnostics.append(Diagnostic("unknown-input-source", f"Unknown input source '{source}'.", item_path + ("type",))) continue + allowed = { + "direct": {"type", "value", "indexed"}, + "local": {"type", "local_name", "indexed"}, + "global": {"type", "global_name", "indexed"}, + "method": {"type", "value", "indexed"}, + }[source] + for field_name in config: + if field_name not in allowed: + diagnostics.append(Diagnostic("unknown-input-field", f"Unknown field '{field_name}' for input source '{source}'.", item_path + (field_name,))) required_key = {"direct": "value", "local": "local_name", "global": "global_name", "method": "value"}[source] if required_key not in config: diagnostics.append(Diagnostic("missing-input-source-value", f"Input source '{source}' requires '{required_key}'.", item_path)) @@ -247,6 +275,15 @@ def _validate_output_mappings(mapping: Mapping[str, Any], path: tuple[str | int, if kind not in {"passfail", "equals", "range", "passthrough", "local", "global", "image"}: diagnostics.append(Diagnostic("unknown-output-type", f"Unknown output type '{kind}'.", item_path + ("type",))) continue + allowed = { + "passfail": {"type"}, "equals": {"type", "value"}, + "range": {"type", "min", "max"}, "passthrough": {"type"}, + "local": {"type", "local_name"}, "global": {"type", "global_name"}, + "image": {"type"}, + }[kind] + for field_name in config: + if field_name not in allowed: + diagnostics.append(Diagnostic("unknown-output-field", f"Unknown field '{field_name}' for output type '{kind}'.", item_path + (field_name,))) if kind in {"passfail", "equals", "range", "passthrough"}: verdicts.append(kind) required = requirements.get(kind, ()) @@ -312,6 +349,10 @@ def validate_recipe_documents(documents: Iterable[Any]) -> ValidationResult: if not isinstance(header, Mapping): return ValidationResult((Diagnostic("invalid-header", "The first document must be the recipe header.", (0,)),)) _check_fields(header, HEADER_FIELDS, (0,), diagnostics) + if header.get("recipe_version") != CANONICAL_RECIPE_VERSION: + diagnostics.append(Diagnostic("unsupported-recipe-version", f"recipe_version must be '{CANONICAL_RECIPE_VERSION}'.", (0, "recipe_version"))) + if "report" in header and header.get("report") not in {"overwrite", "append"}: + diagnostics.append(Diagnostic("invalid-report-mode", "report must be 'overwrite' or 'append'.", (0, "report"))) sequences: dict[str, Mapping[str, Any]] = {} sequence_step_types: dict[str, dict[str, list[str]]] = {} for doc_index, sequence in enumerate(docs[1:], start=1): diff --git a/src/pypts/recipe_parser.py b/src/pypts/recipe_parser.py new file mode 100644 index 0000000..3d25be4 --- /dev/null +++ b/src/pypts/recipe_parser.py @@ -0,0 +1,617 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Safe, source-aware parsing for the pypts recipe language. + +The parser is intentionally isolated from recipe execution and GUI code. It +turns YAML into immutable definitions after validation by +``pypts.recipe_language``. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any, TypeAlias + +import yaml + +from pypts.recipe_language import ( + Diagnostic, + SourcePosition, + SourceSpan, + STEP_SPECS_BY_NAME, + canonical_step_type, + validate_recipe_documents, +) + + +RecipePath: TypeAlias = tuple[str | int, ...] + + +def _freeze(value: Any) -> Any: + if isinstance(value, Mapping): + return FrozenMap((str(key), _freeze(item)) for key, item in value.items()) + if isinstance(value, list | tuple): + return tuple(_freeze(item) for item in value) + if isinstance(value, set | frozenset): + return frozenset(_freeze(item) for item in value) + return value + + +def _thaw(value: Any) -> Any: + if isinstance(value, FrozenMap): + return {key: _thaw(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_thaw(item) for item in value] + if isinstance(value, frozenset): + return set(_thaw(item) for item in value) + return value + + +@dataclass(frozen=True, eq=False) +class FrozenMap(Mapping[str, Any]): + """Small insertion-ordered immutable mapping used by parsed models.""" + + entries: tuple[tuple[str, Any], ...] = () + + def __init__(self, entries=()): + object.__setattr__(self, "entries", tuple(entries)) + + @classmethod + def from_mapping(cls, value: Mapping[str, Any] | None) -> "FrozenMap": + return cls((str(key), _freeze(item)) for key, item in (value or {}).items()) + + def __getitem__(self, key: str) -> Any: + for candidate, value in self.entries: + if candidate == key: + return value + raise KeyError(key) + + def __iter__(self) -> Iterator[str]: + return (key for key, _ in self.entries) + + def __len__(self) -> int: + return len(self.entries) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Mapping) or len(self) != len(other): + return False + return all(key in other and value == other[key] for key, value in self.items()) + + def __hash__(self) -> int: + return hash(frozenset(self.entries)) + + +@dataclass(frozen=True) +class DirectInput: + value: Any + indexed: bool = False + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class LocalInput: + local_name: str + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class GlobalInput: + global_name: str + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class MethodInput: + value: Any + span: SourceSpan | None = field(default=None, compare=False) + + +InputDefinition: TypeAlias = DirectInput | LocalInput | GlobalInput | MethodInput + + +@dataclass(frozen=True) +class PassFailOutput: + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class EqualsOutput: + value: Any + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class RangeOutput: + minimum: Any + maximum: Any + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class PassthroughOutput: + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class LocalOutput: + local_name: str + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class GlobalOutput: + global_name: str + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class ImageOutput: + span: SourceSpan | None = field(default=None, compare=False) + + +OutputDefinition: TypeAlias = ( + PassFailOutput | EqualsOutput | RangeOutput | PassthroughOutput | + LocalOutput | GlobalOutput | ImageOutput +) + + +@dataclass(frozen=True) +class StepDefinition: + steptype: str + step_name: str + description: str + id: str | None + skip: bool + critical: bool + continue_on_error: bool + input_mapping: FrozenMap + output_mapping: FrozenMap + configuration: FrozenMap + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class SequenceDefinition: + sequence_name: str + description: str + parameters: FrozenMap + outputs: FrozenMap + locals: FrozenMap + setup_steps: tuple[StepDefinition, ...] + steps: tuple[StepDefinition, ...] + teardown_steps: tuple[StepDefinition, ...] + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class RecipeHeader: + name: str + version: str + recipe_version: str + description: str + main_sequence: str + globals: FrozenMap + continue_on_error: bool | None + report: str + report_name_include_serial: bool + test_package: str | None + span: SourceSpan | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class RecipeDefinition: + header: RecipeHeader + sequences: tuple[SequenceDefinition, ...] + source_name: str = field(default="", compare=False) + span: SourceSpan | None = field(default=None, compare=False) + + +class RecipeParseError(ValueError): + """Raised when a caller requires a recipe from an unsuccessful parse.""" + + def __init__(self, diagnostics: tuple[Diagnostic, ...]): + self.diagnostics = diagnostics + errors = sum(item.severity == "error" for item in diagnostics) + super().__init__(f"Recipe parsing failed with {errors} error(s).") + + +@dataclass(frozen=True) +class ParseResult: + recipe: RecipeDefinition | None + diagnostics: tuple[Diagnostic, ...] = () + + @property + def errors(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "error") + + @property + def warnings(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "warning") + + @property + def is_valid(self) -> bool: + return self.recipe is not None and not self.errors + + def require_recipe(self) -> RecipeDefinition: + if self.recipe is None or self.errors: + raise RecipeParseError(self.diagnostics) + return self.recipe + + +def _position(mark: yaml.error.Mark) -> SourcePosition: + return SourcePosition(mark.line + 1, mark.column + 1, mark.index) + + +def _span(node: yaml.Node) -> SourceSpan: + return SourceSpan(_position(node.start_mark), _position(node.end_mark)) + + +def _mark_span(mark: yaml.error.Mark | None) -> SourceSpan | None: + if mark is None: + return None + position = _position(mark) + return SourceSpan(position, position) + + +def _index_nodes( + node: yaml.Node, + path: RecipePath, + spans: dict[RecipePath, SourceSpan], + diagnostics: list[Diagnostic], + source_name: str, + active: set[int], +) -> None: + spans[path] = _span(node) + node_id = id(node) + if node_id in active: + diagnostics.append(Diagnostic( + "recursive-alias", "Recursive YAML aliases are not supported.", path, + source_name=source_name, span=_span(node), + )) + return + active.add(node_id) + try: + if isinstance(node, yaml.MappingNode): + seen: set[tuple[str, str]] = set() + for key_node, value_node in node.value: + if isinstance(key_node, yaml.ScalarNode): + identity = (key_node.tag, key_node.value) + key: str | int = key_node.value + else: + identity = (key_node.tag, repr(key_node.value)) + key = repr(key_node.value) + child_path = path + (key,) + if identity in seen: + diagnostics.append(Diagnostic( + "duplicate-key", f"Duplicate YAML key '{key}'.", child_path, + source_name=source_name, span=_span(key_node), + )) + seen.add(identity) + spans[child_path] = _span(value_node) + _index_nodes(value_node, child_path, spans, diagnostics, source_name, active) + elif isinstance(node, yaml.SequenceNode): + for index, child in enumerate(node.value): + _index_nodes(child, path + (index,), spans, diagnostics, source_name, active) + finally: + active.remove(node_id) + + +def _nearest_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: + candidate = path + while candidate: + if candidate in spans: + return spans[candidate] + candidate = candidate[:-1] + return spans.get(()) + + +def _source_diagnostic(code: str, message: str, source_name: str, mark=None) -> Diagnostic: + return Diagnostic(code, message, source_name=source_name, span=_mark_span(mark)) + + +def _enrich_diagnostics( + diagnostics: tuple[Diagnostic, ...], + source_name: str, + spans: Mapping[RecipePath, SourceSpan], + sequence_documents: Mapping[str, int], +) -> list[Diagnostic]: + enriched: list[Diagnostic] = [] + for item in diagnostics: + path = item.path + if path and isinstance(path[0], str) and path[0] in sequence_documents: + path = (sequence_documents[path[0]],) + path[1:] + enriched.append(replace( + item, + source_name=item.source_name or source_name, + span=item.span or _nearest_span(path, spans), + )) + return enriched + + +def _normalization_warnings( + documents: list[Any], + source_name: str, + spans: Mapping[RecipePath, SourceSpan], +) -> list[Diagnostic]: + warnings: list[Diagnostic] = [] + for doc_index, document in enumerate(documents[1:], start=1): + if not isinstance(document, Mapping): + continue + for section in ("setup_steps", "steps", "teardown_steps"): + values = document.get(section, []) + if not isinstance(values, list): + continue + for step_index, step in enumerate(values): + if not isinstance(step, Mapping): + continue + step_path = (doc_index, section, step_index) + raw_type = step.get("steptype") + canonical = canonical_step_type(raw_type) + if canonical and raw_type != canonical: + path = step_path + ("steptype",) + warnings.append(Diagnostic( + "noncanonical-step-type", + f"Use canonical step type '{canonical}' instead of '{raw_type}'.", + path, "warning", source_name, _nearest_span(path, spans), + )) + mapping = step.get("input_mapping", {}) + if isinstance(mapping, Mapping): + for input_name, config in mapping.items(): + if isinstance(config, Mapping) and "type" not in config: + path = step_path + ("input_mapping", input_name) + warnings.append(Diagnostic( + "implicit-direct-input", + f"Input '{input_name}' omits type; it is normalized to 'direct'.", + path, "warning", source_name, _nearest_span(path, spans), + )) + return warnings + + +def _mapping_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: + return _nearest_span(path, spans) + + +def _build_input(config: Mapping[str, Any], path: RecipePath, spans) -> InputDefinition: + kind = config.get("type", "direct") + item_span = _mapping_span(path, spans) + if kind == "direct": + return DirectInput(_freeze(config["value"]), bool(config.get("indexed", False)), item_span) + if kind == "local": + return LocalInput(config["local_name"], item_span) + if kind == "global": + return GlobalInput(config["global_name"], item_span) + return MethodInput(_freeze(config["value"]), item_span) + + +def _build_output(config: Mapping[str, Any], path: RecipePath, spans) -> OutputDefinition: + kind = config["type"] + item_span = _mapping_span(path, spans) + if kind == "passfail": + return PassFailOutput(item_span) + if kind == "equals": + return EqualsOutput(_freeze(config["value"]), item_span) + if kind == "range": + return RangeOutput(_freeze(config["min"]), _freeze(config["max"]), item_span) + if kind == "passthrough": + return PassthroughOutput(item_span) + if kind == "local": + return LocalOutput(config["local_name"], item_span) + if kind == "global": + return GlobalOutput(config["global_name"], item_span) + return ImageOutput(item_span) + + +_COMMON_STEP_KEYS = { + "steptype", "step_name", "description", "id", "skip", "critical", + "continue_on_error", "input_mapping", "output_mapping", +} + + +def _build_step(step: Mapping[str, Any], path: RecipePath, spans) -> StepDefinition: + canonical = canonical_step_type(step["steptype"]) + assert canonical is not None + inputs = FrozenMap( + (str(name), _build_input(config, path + ("input_mapping", name), spans)) + for name, config in step.get("input_mapping", {}).items() + ) + outputs = FrozenMap( + (str(name), _build_output(config, path + ("output_mapping", name), spans)) + for name, config in step.get("output_mapping", {}).items() + ) + configuration = FrozenMap( + (name, _freeze(value)) for name, value in step.items() + if name not in _COMMON_STEP_KEYS + ) + return StepDefinition( + canonical, step["step_name"], step["description"], step.get("id"), + bool(step.get("skip", False)), bool(step.get("critical", False)), + bool(step.get("continue_on_error", False)), inputs, outputs, + configuration, _mapping_span(path, spans), + ) + + +def _build_sequence(sequence: Mapping[str, Any], doc_index: int, spans) -> SequenceDefinition: + def build_section(name: str) -> tuple[StepDefinition, ...]: + return tuple( + _build_step(step, (doc_index, name, index), spans) + for index, step in enumerate(sequence.get(name, [])) + ) + return SequenceDefinition( + sequence["sequence_name"], sequence["description"], + FrozenMap.from_mapping(sequence["parameters"]), + FrozenMap.from_mapping(sequence["outputs"]), + FrozenMap.from_mapping(sequence["locals"]), + build_section("setup_steps"), build_section("steps"), + build_section("teardown_steps"), _mapping_span((doc_index,), spans), + ) + + +def _build_recipe(documents: list[Mapping[str, Any]], source_name: str, spans) -> RecipeDefinition: + raw = documents[0] + header = RecipeHeader( + raw["name"], raw["version"], raw["recipe_version"], raw["description"], + raw["main_sequence"], FrozenMap.from_mapping(raw["globals"]), + raw.get("continue_on_error"), raw.get("report", "overwrite"), + bool(raw.get("report_name_include_serial", False)), raw.get("test_package"), + _mapping_span((0,), spans), + ) + sequences = tuple( + _build_sequence(sequence, index, spans) + for index, sequence in enumerate(documents[1:], start=1) + ) + recipe_span = None + if documents: + first = spans.get((0,)) + last = spans.get((len(documents) - 1,)) + if first and last: + recipe_span = SourceSpan(first.start, last.end) + return RecipeDefinition(header, sequences, source_name, recipe_span) + + +def parse_recipe_text(text: str, source_name: str = "") -> ParseResult: + """Parse recipe YAML text without importing or invoking the runtime.""" + if not isinstance(text, str): + diagnostic = Diagnostic("invalid-source", "Recipe source must be text.", source_name=source_name) + return ParseResult(None, (diagnostic,)) + if not text.strip(): + diagnostic = Diagnostic("empty-recipe", "A recipe requires a header and at least one sequence.", source_name=source_name) + return ParseResult(None, (diagnostic,)) + + try: + nodes = list(yaml.compose_all(text, Loader=yaml.SafeLoader)) + except yaml.YAMLError as error: + mark = getattr(error, "problem_mark", None) + return ParseResult(None, (_source_diagnostic("yaml-syntax-error", str(error), source_name, mark),)) + + spans: dict[RecipePath, SourceSpan] = {} + diagnostics: list[Diagnostic] = [] + for index, node in enumerate(nodes): + if node is not None: + _index_nodes(node, (index,), spans, diagnostics, source_name, set()) + + try: + documents = list(yaml.safe_load_all(text)) + except yaml.YAMLError as error: + mark = getattr(error, "problem_mark", None) + code = "unsafe-yaml" if isinstance(error, yaml.constructor.ConstructorError) else "yaml-construction-error" + diagnostics.append(_source_diagnostic(code, str(error), source_name, mark)) + return ParseResult(None, tuple(diagnostics)) + + sequence_documents = { + document["sequence_name"]: index + for index, document in enumerate(documents) + if isinstance(document, Mapping) and isinstance(document.get("sequence_name"), str) + } + contract = validate_recipe_documents(documents) + diagnostics.extend(_enrich_diagnostics(contract.diagnostics, source_name, spans, sequence_documents)) + diagnostics.extend(_normalization_warnings(documents, source_name, spans)) + if any(item.severity == "error" for item in diagnostics): + return ParseResult(None, tuple(diagnostics)) + return ParseResult(_build_recipe(documents, source_name, spans), tuple(diagnostics)) + + +def parse_recipe_file(path: str | Path, encoding: str = "utf-8") -> ParseResult: + """Read and parse a recipe file, reporting read failures as diagnostics.""" + source_path = Path(path) + source_name = str(source_path) + try: + text = source_path.read_text(encoding=encoding) + except (OSError, UnicodeError) as error: + return ParseResult(None, (Diagnostic( + "file-read-error", f"Could not read recipe: {error}", + source_name=source_name, + ),)) + return parse_recipe_text(text, source_name) + + +def _input_to_mapping(value: InputDefinition) -> dict[str, Any]: + if isinstance(value, DirectInput): + result = {"type": "direct", "value": _thaw(value.value)} + if value.indexed: + result["indexed"] = True + return result + if isinstance(value, LocalInput): + return {"type": "local", "local_name": value.local_name} + if isinstance(value, GlobalInput): + return {"type": "global", "global_name": value.global_name} + return {"type": "method", "value": _thaw(value.value)} + + +def _output_to_mapping(value: OutputDefinition) -> dict[str, Any]: + if isinstance(value, PassFailOutput): + return {"type": "passfail"} + if isinstance(value, EqualsOutput): + return {"type": "equals", "value": _thaw(value.value)} + if isinstance(value, RangeOutput): + return {"type": "range", "min": _thaw(value.minimum), "max": _thaw(value.maximum)} + if isinstance(value, PassthroughOutput): + return {"type": "passthrough"} + if isinstance(value, LocalOutput): + return {"type": "local", "local_name": value.local_name} + if isinstance(value, GlobalOutput): + return {"type": "global", "global_name": value.global_name} + return {"type": "image"} + + +def _step_to_mapping(step: StepDefinition) -> dict[str, Any]: + result: dict[str, Any] = {"steptype": step.steptype, "step_name": step.step_name} + if step.id is not None: + result["id"] = step.id + result["description"] = step.description + result["skip"] = step.skip + result["critical"] = step.critical + result["continue_on_error"] = step.continue_on_error + spec = STEP_SPECS_BY_NAME[step.steptype.casefold()] + for field_spec in spec.fields: + if field_spec.name in _COMMON_STEP_KEYS or field_spec.name not in step.configuration: + continue + result[field_spec.name] = _thaw(step.configuration[field_spec.name]) + result["input_mapping"] = { + name: _input_to_mapping(value) for name, value in step.input_mapping.items() + } + result["output_mapping"] = { + name: _output_to_mapping(value) for name, value in step.output_mapping.items() + } + return result + + +def _sequence_to_mapping(sequence: SequenceDefinition) -> dict[str, Any]: + return { + "sequence_name": sequence.sequence_name, + "description": sequence.description, + "parameters": _thaw(sequence.parameters), + "outputs": _thaw(sequence.outputs), + "locals": _thaw(sequence.locals), + "setup_steps": [_step_to_mapping(step) for step in sequence.setup_steps], + "steps": [_step_to_mapping(step) for step in sequence.steps], + "teardown_steps": [_step_to_mapping(step) for step in sequence.teardown_steps], + } + + +def dump_recipe(recipe: RecipeDefinition) -> str: + """Serialize a typed recipe to stable canonical multi-document YAML.""" + if not isinstance(recipe, RecipeDefinition): + raise TypeError("dump_recipe expects a RecipeDefinition") + header = recipe.header + header_document: dict[str, Any] = { + "name": header.name, + "version": header.version, + "recipe_version": header.recipe_version, + "description": header.description, + "main_sequence": header.main_sequence, + } + if header.test_package is not None: + header_document["test_package"] = header.test_package + if header.continue_on_error is not None: + header_document["continue_on_error"] = header.continue_on_error + header_document["report"] = header.report + header_document["report_name_include_serial"] = header.report_name_include_serial + header_document["globals"] = _thaw(header.globals) + documents = [header_document] + [_sequence_to_mapping(sequence) for sequence in recipe.sequences] + return yaml.safe_dump_all( + documents, explicit_start=True, sort_keys=False, + default_flow_style=False, allow_unicode=True, + ) diff --git a/tests/unit_tests/test_recipe_parser.py b/tests/unit_tests/test_recipe_parser.py new file mode 100644 index 0000000..b971e47 --- /dev/null +++ b/tests/unit_tests/test_recipe_parser.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later + +import ast +from pathlib import Path +import textwrap + +import pytest + +from pypts.recipe_parser import ( + DirectInput, + EqualsOutput, + GlobalInput, + GlobalOutput, + ImageOutput, + LocalInput, + LocalOutput, + MethodInput, + PassFailOutput, + PassthroughOutput, + RangeOutput, + RecipeParseError, + dump_recipe, + parse_recipe_file, + parse_recipe_text, +) + + +RECIPES = Path(__file__).parents[2] / "src" / "pypts" / "recipes" +PARSER = Path(__file__).parents[2] / "src" / "pypts" / "recipe_parser.py" + + +def recipe_text(steps="[]", *, recipe_version="1.0.0"): + raw_steps = textwrap.dedent(steps).strip() + steps_value = f" {raw_steps}" if raw_steps == "[]" else "\n" + textwrap.indent(raw_steps, " ") + return f"""--- +name: Parser test +version: "1" +recipe_version: {recipe_version} +description: Parser test recipe +main_sequence: Main +globals: {{}} +--- +sequence_name: Main +description: Main sequence +parameters: {{}} +outputs: {{}} +locals: {{}} +setup_steps: [] +steps:{steps_value} +teardown_steps: [] +""" + + +def test_parse_builds_typed_mappings_and_defaults(): + steps = """ + - steptype: PythonModuleStep + step_name: typed mappings + description: Exercise all mapping models + action_type: method + module: tests.py + method_name: run + input_mapping: + direct: {type: direct, value: [1, 2], indexed: true} + local: {type: local, local_name: local_value} + global: {type: global, global_name: global_value} + method: {type: method, value: helper} + output_mapping: + passed: {type: passfail} + exact: {type: equals, value: 3} + bounded: {type: range, min: 1, max: 4} + local: {type: local, local_name: saved} + global: {type: global, global_name: saved} + chart: {type: image} + - steptype: SequenceStep + step_name: passthrough + description: Exercise passthrough + sequence: {type: internal, name: Main} + output_mapping: + result: {type: passthrough} + """ + result = parse_recipe_text(recipe_text(steps)) + recipe = result.require_recipe() + first, second = recipe.sequences[0].steps + + assert first.skip is first.critical is first.continue_on_error is False + assert isinstance(first.input_mapping["direct"], DirectInput) + assert isinstance(first.input_mapping["local"], LocalInput) + assert isinstance(first.input_mapping["global"], GlobalInput) + assert isinstance(first.input_mapping["method"], MethodInput) + assert isinstance(first.output_mapping["passed"], PassFailOutput) + assert isinstance(first.output_mapping["exact"], EqualsOutput) + assert isinstance(first.output_mapping["bounded"], RangeOutput) + assert isinstance(first.output_mapping["local"], LocalOutput) + assert isinstance(first.output_mapping["global"], GlobalOutput) + assert isinstance(first.output_mapping["chart"], ImageOutput) + assert isinstance(second.output_mapping["result"], PassthroughOutput) + + +def test_normalization_warnings_and_canonical_dump(): + steps = """ + - steptype: waitstep + step_name: wait + description: Legacy spelling + input_mapping: + wait_time: {value: 1} + """ + result = parse_recipe_text(recipe_text(steps), "legacy.yml") + assert {warning.code for warning in result.warnings} == { + "noncanonical-step-type", "implicit-direct-input", + } + canonical = dump_recipe(result.require_recipe()) + assert "steptype: WaitStep" in canonical + assert "type: direct" in canonical + assert "skip: false" in canonical + assert not parse_recipe_text(canonical).warnings + + +def test_contract_diagnostic_has_source_and_nearest_span(): + text = recipe_text("[]").replace("main_sequence: Main", "main_sequence: Missing") + result = parse_recipe_text(text, "broken.yml") + diagnostic = next(item for item in result.errors if item.code == "unknown-main-sequence") + assert diagnostic.source_name == "broken.yml" + assert diagnostic.span is not None + expected_line = next(index for index, line in enumerate(text.splitlines(), start=1) if line.startswith("main_sequence:")) + assert diagnostic.span.start.line == expected_line + assert diagnostic.span.start.column > 1 + + +def test_missing_field_uses_parent_span(): + text = recipe_text("[]").replace("description: Main sequence\n", "") + result = parse_recipe_text(text, "missing.yml") + diagnostic = next(item for item in result.errors if item.code == "missing-field") + assert diagnostic.path[-1] == "description" + assert diagnostic.span is not None + assert diagnostic.span.start.line > 1 + + +def test_duplicate_yaml_keys_are_rejected(): + text = recipe_text("[]").replace("name: Parser test", "name: First\nname: Second") + result = parse_recipe_text(text) + assert "duplicate-key" in {item.code for item in result.errors} + assert result.recipe is None + + +def test_malformed_and_unsafe_yaml_are_rejected(): + malformed = parse_recipe_text("name: [unterminated") + unsafe = parse_recipe_text("!!python/object:builtins.object {}") + assert {item.code for item in malformed.errors} == {"yaml-syntax-error"} + assert "unsafe-yaml" in {item.code for item in unsafe.errors} + + +def test_empty_and_non_text_sources_are_rejected(): + assert {item.code for item in parse_recipe_text(" \n").errors} == {"empty-recipe"} + assert {item.code for item in parse_recipe_text(None).errors} == {"invalid-source"} + + +def test_file_entry_point_and_read_failure(tmp_path): + path = tmp_path / "recipe.yml" + path.write_text(recipe_text("[]"), encoding="utf-8") + from_file = parse_recipe_file(path) + from_text = parse_recipe_text(path.read_text(encoding="utf-8"), str(path)) + assert from_file.recipe == from_text.recipe + assert parse_recipe_file(tmp_path / "missing.yml").errors[0].code == "file-read-error" + + +def test_require_recipe_raises_with_diagnostics(): + result = parse_recipe_text("") + with pytest.raises(RecipeParseError) as error: + result.require_recipe() + assert error.value.diagnostics == result.diagnostics + + +def test_unsupported_recipe_version_is_rejected(): + result = parse_recipe_text(recipe_text("[]", recipe_version="2.0.0")) + assert "unsupported-recipe-version" in {item.code for item in result.errors} + + +@pytest.mark.parametrize( + "path", + sorted(path for path in RECIPES.glob("*.yml") if path.name != "subsequence_executions_draft.yml"), +) +def test_bundled_recipe_parse_dump_reparse_is_model_stable(path): + first = parse_recipe_file(path) + assert first.is_valid, first.errors + canonical = dump_recipe(first.require_recipe()) + second = parse_recipe_text(canonical, f"canonical:{path.name}") + assert second.is_valid, second.errors + assert second.recipe == first.recipe + + +def test_comment_only_draft_is_rejected_as_empty(): + result = parse_recipe_file(RECIPES / "subsequence_executions_draft.yml") + assert {item.code for item in result.errors} == {"empty-recipe"} + + +def test_parser_has_no_runtime_gui_or_docs_imports(): + tree = ast.parse(PARSER.read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} + assert not any(name == item or name.startswith(item + ".") for name in imported for item in forbidden) From a91769bdf1318d0e910460cbd65205fccfc98405 Mon Sep 17 00:00:00 2001 From: alvaro Date: Sat, 8 Aug 2026 15:18:53 +0200 Subject: [PATCH 03/14] docs --- docs/source/index.rst | 1 + docs/source/recipe_language_architecture.rst | 213 +++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 docs/source/recipe_language_architecture.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 3297167..1a361ae 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -56,6 +56,7 @@ Documentation contents api architecture + recipe_language_architecture gui_architecture yaml_format instruments diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst new file mode 100644 index 0000000..bddfeb3 --- /dev/null +++ b/docs/source/recipe_language_architecture.rst @@ -0,0 +1,213 @@ +.. SPDX-FileCopyrightText: 2026 CERN +.. +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Recipe Language Architecture +============================ + +This page describes the recipe language model, the isolated parser, and the +rules for evolving them. The parser is currently independent of recipe +execution, ``verify_recipe``, YamVIEW, and Sphinx reference generation. Those +consumers will move onto the shared model in later integration work. + +Design goals +------------ + +The recipe language has one contract and one parsing path. YAML loading, +language validation, normalized data, and runtime construction are separate +responsibilities. In particular, parsing a recipe must never import or invoke +the runtime, steps, GUI, or documentation machinery. + +The current architecture has two source modules: + +``pypts.recipe_language`` + Defines the framework-independent contract. Its field and step + specifications describe accepted recipe documents, while + ``validate_recipe_documents`` validates already-loaded Python values. It + has no YAML or runtime dependency. + +``pypts.recipe_parser`` + Safely loads YAML, records source locations, delegates language rules to the + contract, and constructs immutable typed definitions. It also serializes a + typed recipe into canonical YAML. + +The intended flow is:: + + recipe YAML + | + v + safe YAML loading and source indexing + | + v + recipe_language contract validation + | + +---- errors and warnings with source spans + | + v + immutable RecipeDefinition + | + +---- dump_recipe() -> canonical recipe YAML + | + +---- future runtime, GUI, and generated-reference consumers + +Parser operation +---------------- + +``parse_recipe_text`` and ``parse_recipe_file`` return a ``ParseResult``. The +parser performs these stages in order: + +#. Reject non-text, empty, or unreadable input. +#. Compose the YAML with ``SafeLoader`` to index nodes and their one-based line + and column positions. Duplicate mapping keys and recursive aliases are + diagnosed here. +#. Safely construct the YAML documents. Malformed YAML, unsafe tags, and + construction failures become diagnostics rather than runtime objects. +#. Pass the loaded documents to ``validate_recipe_documents``. This is where + header, sequence, step, mapping, reference, and lifecycle rules are applied. +#. Attach the closest available source span to each contract diagnostic and + emit warnings for accepted legacy spellings or implicit forms. +#. If any error exists, return no recipe. Otherwise, normalize the documents + into an immutable ``RecipeDefinition``. + +``ParseResult.errors`` and ``ParseResult.warnings`` split diagnostics by +severity. ``ParseResult.require_recipe()`` returns the model on success and +raises ``RecipeParseError`` with all diagnostics on failure. Callers that need +to present every issue should inspect the result before requiring the model. + +Diagnostics contain a stable code, message, semantic path, severity, source +name, and optional ``SourceSpan``. Consumers should branch on the code rather +than matching message text. A source span is half-open; its line and column +values are one-based and its character offset is zero-based. + +Typed and normalized model +-------------------------- + +The model is made of frozen definitions for the recipe header, sequences, +steps, and each input and output mapping variant. Arbitrary mappings that are +part of recipe data use the immutable, insertion-ordered ``FrozenMap``. Source +spans and source names do not participate in semantic equality, which makes a +parse/dump/reparse comparison independent of file location. + +Normalization currently includes: + +* canonical step type casing; +* explicit typed input and output definitions; +* false defaults for step flags; +* empty mappings for optional mapping fields; +* recipe report defaults; and +* removal of runtime-ignored legacy sequence metadata from the typed model. + +``dump_recipe`` emits stable, explicit-start, multi-document YAML. It writes +canonical step names, explicit input types, explicit defaults, and stable field +ordering. It does not preserve comments or the source's original formatting. +The semantic guarantee is therefore model round-trip equality, not textual +round-trip equality. + +Using the parser +---------------- + +Parse in-memory YAML when the caller already owns the text: + +.. code-block:: python + + from pypts.recipe_parser import parse_recipe_text + + result = parse_recipe_text(source, source_name="recipe.yml") + if result.errors: + for diagnostic in result.errors: + print(diagnostic.code, diagnostic.path, diagnostic.span) + else: + recipe = result.require_recipe() + +Use ``parse_recipe_file`` when the parser should read the file and report I/O +or decoding failures as diagnostics. Use ``dump_recipe`` only with a valid +``RecipeDefinition``. + +Architecture boundaries +----------------------- + +Keep the dependency direction narrow: + +* ``recipe_language`` must not depend on YAML, runtime classes, concrete steps, + YamVIEW, or Sphinx. +* ``recipe_parser`` may depend on PyYAML and ``recipe_language`` but not on + runtime, concrete steps, YamVIEW, or Sphinx. +* Runtime and UI adapters may consume parser models after integration; parser + models must not consume those adapters. +* Syntax reference pages and examples should eventually be generated or + checked from the language specifications. Architecture prose should explain + responsibilities and extension workflows, not duplicate field tables. + +These rules keep syntax inspection safe and make the parser usable by command +line tools, editors, the GUI, tests, and documentation without constructing +hardware-facing runtime objects. + +Maintaining the language +------------------------ + +Adding or changing a step +~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Update its ``StepSpec`` and ``FieldSpec`` entries in + ``pypts.recipe_language``. Do not create a second field list in a consumer. +#. Add semantic checks beside the shared contract validation when a constraint + cannot be represented by required fields and value types. +#. Add a valid executable parser fixture and focused invalid cases for every + new constraint. +#. Confirm canonical serialization contains the step-specific configuration in + specification order and parse/dump/reparse preserves the model. +#. During framework integration, update only the adapter that constructs the + runtime step from ``StepDefinition``. +#. Regenerate the syntax reference and validate its example once reference + generation is available. + +Changing input or output mappings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. Update the allowed and required fields in the contract validator. +#. Add or adjust the corresponding frozen mapping definition, model builder, + and serializer branch in ``recipe_parser``. +#. Test accepted values, every relevant failure, source spans, normalization, + and canonical round trips. +#. Check runtime and GUI adapters after integration. They should dispatch on + typed mapping definitions instead of maintaining supported-type lists. + +Evolving ``recipe_version`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Recipe format changes must be explicit. Do not silently reinterpret an +existing version. First describe compatibility and migration behavior, then +add version-specific contract handling and fixtures. Preserve parsing for a +supported old version or emit a precise unsupported-version diagnostic. A +canonical dump must state the version whose semantics it writes. + +Retiring legacy syntax +~~~~~~~~~~~~~~~~~~~~~~ + +Legacy syntax should move through an observable sequence: accept and normalize +with a stable warning, document the canonical replacement, measure and migrate +the bundled examples and consumers, and only then reject it in a declared +recipe version. Never remove an accepted form merely by changing a runtime +constructor. + +Verification +------------ + +The parser and language tests are in ``tests/unit_tests/test_recipe_parser.py`` +and ``tests/unit_tests/test_recipe_language.py``. Run them directly while +editing the contract, then run the complete suite: + +.. code-block:: console + + python -m pytest tests/unit_tests/test_recipe_language.py tests/unit_tests/test_recipe_parser.py + python -m pytest + +The acceptance corpus consists of every non-empty bundled recipe. Each must +parse, dump, reparse, and compare equal as a model. The comment-only draft is +intentionally invalid. Isolation tests also protect the parser from importing +runtime, step, GUI, or Sphinx modules. + +After the integration and generated-reference phases, verification must also +cover runtime construction equivalence, GUI-produced canonical YAML, registry +and reference completeness, successful Sphinx builds with warnings treated as +errors, and the absence of duplicated consumer-side field or type registries. From b99d9fd32ad2ea87cfb3d48bb341586898f3beb3 Mon Sep 17 00:00:00 2001 From: alvaro Date: Tue, 11 Aug 2026 09:14:54 +0200 Subject: [PATCH 04/14] add recipe reference first implementation --- docs/generated/recipe_language_reference.rst | 918 +++++++++++++++++++ docs/source/recipe_language_architecture.rst | 33 +- src/pypts/recipe_language.py | 414 +++++++-- src/pypts/recipe_reference.py | 334 +++++++ tests/unit_tests/test_recipe_reference.py | 261 ++++++ 5 files changed, 1889 insertions(+), 71 deletions(-) create mode 100644 docs/generated/recipe_language_reference.rst create mode 100644 src/pypts/recipe_reference.py create mode 100644 tests/unit_tests/test_recipe_reference.py diff --git a/docs/generated/recipe_language_reference.rst b/docs/generated/recipe_language_reference.rst new file mode 100644 index 0000000..9028cc9 --- /dev/null +++ b/docs/generated/recipe_language_reference.rst @@ -0,0 +1,918 @@ +.. SPDX-FileCopyrightText: 2026 CERN +.. +.. SPDX-License-Identifier: CC-BY-SA-4.0 +.. +.. This file is generated by pypts.recipe_reference. Do not edit it manually. + +Recipe Language Reference +========================= + +Canonical recipe language version: ``1.0.0``. + +Document grammar +---------------- + +A recipe is safe multi-document YAML. The first document is one recipe +header and every following document is one sequence. At least one sequence +is required, and ``main_sequence`` must name one of them. + +Recipe header +------------- + +The first YAML document; identifies the recipe and its entry sequence. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``name`` + - str + - required + - Human-readable recipe name. + * - ``version`` + - str + - required + - Version of this recipe. + * - ``recipe_version`` + - str + - required; allowed: ``1.0.0`` + - Version of the recipe language contract. + * - ``description`` + - str + - required + - Purpose of the recipe. + * - ``main_sequence`` + - str + - required + - Sequence where execution begins. + * - ``globals`` + - dict + - required + - Recipe-wide variables. + * - ``continue_on_error`` + - bool + - optional; default: ``null`` + - Recipe-wide error policy. + * - ``report`` + - str + - optional; default: ``overwrite``; allowed: ``overwrite``, ``append`` + - Report file mode. + * - ``report_name_include_serial`` + - bool + - optional; default: ``false`` + - Include the serial number in the report name. + * - ``test_package`` + - str + - optional; default: ``null`` + - Package containing recipe test modules. + +Sequence +-------- + +Each YAML document after the header defines one named sequence. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``sequence_name`` + - str + - required + - Unique sequence name. + * - ``description`` + - str + - required + - Purpose of the sequence. + * - ``parameters`` + - dict + - required + - Reserved sequence input metadata. + * - ``outputs`` + - dict + - required + - Reserved sequence output metadata. + * - ``locals`` + - dict + - required + - Variables local to the sequence. + * - ``setup_steps`` + - list + - required + - Steps run before the main steps. + * - ``steps`` + - list + - required + - Ordered main steps. + * - ``teardown_steps`` + - list + - required + - Steps run during teardown. + * - ``serial_number`` + - str or int + - optional; legacy + - Runtime-ignored legacy sequence metadata. + +Common step fields +------------------ + +These fields are shared by every authorable step type. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``steptype`` + - str + - required + - Registered step type. + * - ``step_name`` + - str + - required + - Human-readable step name. + * - ``description`` + - str + - required + - Purpose of the step. + * - ``id`` + - str + - optional + - Optional stable step identifier. + * - ``skip`` + - bool + - optional; default: ``false`` + - Skip execution. + * - ``critical`` + - bool + - optional; default: ``false`` + - Stop on error when policy permits continuation. + * - ``continue_on_error`` + - bool + - optional; default: ``false`` + - Per-step error policy. + * - ``input_mapping`` + - dict + - optional; default: ``{}`` + - Named input sources. + * - ``output_mapping`` + - dict + - optional; default: ``{}`` + - Named verdicts and destinations. + +Registered step types +--------------------- + +.. _recipe-step-pythonmodulestep: + +PythonModuleStep +~~~~~~~~~~~~~~~~ + +Calls a method or reads/writes an attribute in a Python module. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``action_type`` + - str + - required; allowed: ``method``, ``read_attribute``, ``write_attribute`` + - Operation performed on the Python module. + * - ``module`` + - str + - required + - Python module path. + * - ``method_name`` + - str + - optional + - Method name required by method actions. + +Canonical example: + +.. code-block:: yaml + + steptype: PythonModuleStep + step_name: Run test + description: Run a Python test method. + skip: false + critical: false + continue_on_error: false + action_type: method + module: tests.py + method_name: run + input_mapping: {} + output_mapping: {} + +.. _recipe-step-sequencestep: + +SequenceStep +~~~~~~~~~~~~ + +Runs another sequence as a step. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``sequence`` + - dict + - required + - Internal sequence reference. + +Canonical example: + +.. code-block:: yaml + + steptype: SequenceStep + step_name: Run calibration + description: Run an internal sequence. + skip: false + critical: false + continue_on_error: false + sequence: + type: internal + name: Calibration + input_mapping: {} + output_mapping: {} + +.. _recipe-step-userinteractionstep: + +UserInteractionStep +~~~~~~~~~~~~~~~~~~~ + +Displays an operator interaction prompt. + +Canonical example: + +.. code-block:: yaml + + steptype: UserInteractionStep + step_name: Confirm + description: Ask the operator to confirm. + skip: false + critical: false + continue_on_error: false + input_mapping: + message: + type: direct + value: Continue? + output_mapping: + output: + type: passfail + +.. _recipe-step-waitstep: + +WaitStep +~~~~~~~~ + +Waits for a non-negative duration in seconds. + +Required input names: ``wait_time``. + +Canonical example: + +.. code-block:: yaml + + steptype: WaitStep + step_name: Stabilize + description: Wait for hardware stabilization. + skip: false + critical: false + continue_on_error: false + input_mapping: + wait_time: + type: direct + value: 1 + output_mapping: {} + +.. _recipe-step-userloadingstep: + +UserLoadingStep +~~~~~~~~~~~~~~~ + +Prompts the operator to select a file. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``file_save_location`` + - dict + - optional + - Local or global destination for the selected file. + +Canonical example: + +.. code-block:: yaml + + steptype: UserLoadingStep + step_name: Load configuration + description: Ask the operator for a file. + skip: false + critical: false + continue_on_error: false + input_mapping: + message: + type: direct + value: Choose a file + output_mapping: + output: + type: passfail + +.. _recipe-step-userrunmethodstep: + +UserRunMethodStep +~~~~~~~~~~~~~~~~~ + +Optionally runs a Python method after an operator response. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``trigger_response`` + - str or list or dict + - optional + - Operator response that triggers execution. + * - ``action_type`` + - str + - optional + - Optional Python action type. + * - ``module`` + - str + - optional + - Optional Python module path. + * - ``method_name`` + - str + - optional + - Optional Python method name. + +Canonical example: + +.. code-block:: yaml + + steptype: UserRunMethodStep + step_name: Run calibration + description: Run on operator confirmation. + skip: false + critical: false + continue_on_error: false + trigger_response: run + action_type: method + module: tests.py + method_name: calibrate + input_mapping: {} + output_mapping: + output: + type: passfail + +.. _recipe-step-userwritestep: + +UserWriteStep +~~~~~~~~~~~~~ + +Writes an operator-provided value to a configured destination. + +Canonical example: + +.. code-block:: yaml + + steptype: UserWriteStep + step_name: Enter value + description: Ask the operator for a value. + skip: false + critical: false + continue_on_error: false + input_mapping: + message: + type: direct + value: Enter value + output_mapping: + output: + type: local + local_name: value + +.. _recipe-step-serialnumberstep: + +SerialNumberStep +~~~~~~~~~~~~~~~~ + +Captures the device serial number. + +Canonical example: + +.. code-block:: yaml + + steptype: SerialNumberStep + step_name: Scan serial number + description: Capture the device serial number. + skip: false + critical: false + continue_on_error: false + input_mapping: {} + output_mapping: {} + +.. _recipe-step-sshconnectstep: + +SSHConnectStep +~~~~~~~~~~~~~~ + +Opens the SSH client stored in recipe globals. + +Canonical example: + +.. code-block:: yaml + + steptype: SSHConnectStep + step_name: Connect + description: Open the SSH connection. + skip: false + critical: false + continue_on_error: false + input_mapping: {} + output_mapping: {} + +.. _recipe-step-sshclosestep: + +SSHCloseStep +~~~~~~~~~~~~ + +Closes the SSH client stored in recipe globals. + +Canonical example: + +.. code-block:: yaml + + steptype: SSHCloseStep + step_name: Disconnect + description: Close the SSH connection. + skip: false + critical: false + continue_on_error: false + input_mapping: {} + output_mapping: {} + +.. _recipe-step-sshuploadstep: + +SSHUploadStep +~~~~~~~~~~~~~ + +Uploads files through an SSH connection. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``files`` + - list + - required + - Local and remote file pairs to upload. + * - ``permissions`` + - int or str + - optional + - Optional remote permissions. + * - ``skip_if_sha256_match`` + - bool + - optional; default: ``false`` + - Skip files whose remote checksum matches. + * - ``local_package`` + - str + - optional + - Optional package containing local resources. + +Canonical example: + +.. code-block:: yaml + + steptype: SSHUploadStep + step_name: Deploy + description: Upload a file to the target. + skip: false + critical: false + continue_on_error: false + files: + - local: bin/tool + remote: /tmp/tool + input_mapping: {} + output_mapping: + passed: + type: passfail + +Input mapping types +------------------- + +.. _recipe-input-direct: + +**``direct``** + +Provides a literal value. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - optional; default: ``direct``; allowed: ``direct`` + - Input source type. + * - ``value`` + - any + - required + - Literal input value. + * - ``indexed`` + - bool + - optional; default: ``false`` + - Expand a list into indexed steps. + +Canonical mapping: + +.. code-block:: yaml + + type: direct + value: 1 + +.. _recipe-input-local: + +**``local``** + +Reads a sequence-local variable. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``local`` + - Input source type. + * - ``local_name`` + - str + - required + - Local variable name. + * - ``indexed`` + - bool + - optional; default: ``false``; legacy + - Compatibility field; only false is accepted. + +Canonical mapping: + +.. code-block:: yaml + + type: local + local_name: local_value + +.. _recipe-input-global: + +**``global``** + +Reads a recipe-global variable. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``global`` + - Input source type. + * - ``global_name`` + - str + - required + - Global variable name. + * - ``indexed`` + - bool + - optional; default: ``false``; legacy + - Compatibility field; only false is accepted. + +Canonical mapping: + +.. code-block:: yaml + + type: global + global_name: global_value + +.. _recipe-input-method: + +**``method``** + +Resolves a method reference for the step. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``method`` + - Input source type. + * - ``value`` + - any + - required + - Method reference. + * - ``indexed`` + - bool + - optional; default: ``false``; legacy + - Compatibility field; only false is accepted. + +Canonical mapping: + +.. code-block:: yaml + + type: method + value: helper + +Output mapping types +-------------------- + +.. _recipe-output-passfail: + +**``passfail``** + +Interprets the output as a pass/fail verdict. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``passfail`` + - Output mapping type. + +Canonical mapping: + +.. code-block:: yaml + + type: passfail + +.. _recipe-output-equals: + +**``equals``** + +Passes when the output equals the configured value. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``equals`` + - Output mapping type. + * - ``value`` + - any + - required + - Expected value. + +Canonical mapping: + +.. code-block:: yaml + + type: equals + value: 3 + +.. _recipe-output-range: + +**``range``** + +Passes when the output is within an inclusive range. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``range`` + - Output mapping type. + * - ``min`` + - any + - required + - Minimum accepted value. + * - ``max`` + - any + - required + - Maximum accepted value. + +Canonical mapping: + +.. code-block:: yaml + + type: range + min: 1 + max: 4 + +.. _recipe-output-passthrough: + +**``passthrough``** + +Uses the nested result without adding a verdict. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``passthrough`` + - Output mapping type. + +Canonical mapping: + +.. code-block:: yaml + + type: passthrough + +.. _recipe-output-local: + +**``local``** + +Stores the output in a sequence-local variable. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``local`` + - Output mapping type. + * - ``local_name`` + - str + - required + - Local destination variable. + +Canonical mapping: + +.. code-block:: yaml + + type: local + local_name: saved + +.. _recipe-output-global: + +**``global``** + +Stores the output in a recipe-global variable. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``global`` + - Output mapping type. + * - ``global_name`` + - str + - required + - Global destination variable. + +Canonical mapping: + +.. code-block:: yaml + + type: global + global_name: saved + +.. _recipe-output-image: + +**``image``** + +Publishes an image output for presentation. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 28 40 + + * - Field + - Type + - Requirement + - Description + * - ``type`` + - str + - required; allowed: ``image`` + - Output mapping type. + +Canonical mapping: + +.. code-block:: yaml + + type: image + +Semantic constraints +-------------------- + +* ``parser`` — Sources must be readable text and safe, non-recursive YAML without duplicate keys. + Diagnostics: ``invalid-source``, ``file-read-error``, ``yaml-syntax-error``, ``yaml-construction-error``, ``unsafe-yaml``, ``recursive-alias``, ``duplicate-key``. +* ``documents and steps`` — Only declared fields, field types, and allowed values are accepted. + Diagnostics: ``missing-field``, ``invalid-field-type``, ``invalid-field-value``, ``unknown-field``, ``unsupported-recipe-version``, ``invalid-report-mode``, ``invalid-action-type``. +* ``input and output mappings`` — Mapping variants accept only their declared fields and require their declared values. + Diagnostics: ``invalid-input-mapping``, ``unknown-input-source``, ``unknown-input-field``, ``invalid-input-field-type``, ``missing-input-source-value``, ``invalid-output-mapping``, ``unknown-output-type``, ``unknown-output-field``, ``invalid-output-field-type``, ``missing-output-field``. +* ``recipe`` — A recipe is safe multi-document YAML with one header followed by at least one sequence. + Diagnostics: ``empty-recipe``, ``invalid-header``, ``invalid-sequence``. +* ``step`` — Every step is a mapping with a registered step type. + Diagnostics: ``invalid-step``, ``unknown-step-type``. +* ``sequence`` — Sequence names are unique and main_sequence names an existing sequence. + Diagnostics: ``duplicate-sequence``, ``unknown-main-sequence``. +* ``SequenceStep`` — Only internal references are accepted and the named target sequence must exist. + Diagnostics: ``invalid-sequence-reference``, ``unknown-sequence-reference``. +* ``PythonModuleStep`` — A method action requires method_name. + Diagnostics: ``missing-method-name``. +* ``UserLoadingStep`` — A file destination names a local or global variable. + Diagnostics: ``invalid-file-save-location``. +* ``input mapping`` — Indexed inputs are direct lists and all indexed lists on a step have equal length. + Diagnostics: ``invalid-indexed-flag``, ``invalid-indexed-input``, ``unequal-indexed-inputs``. +* ``output mapping`` — passthrough must be the only verdict mapping on its step. + Diagnostics: ``mixed-passthrough``. +* ``step`` — Step-specific required input names must be present. + Diagnostics: ``missing-required-input``, ``missing-input-mapping``. +* ``SSH steps`` — SSH steps require connection globals; setup connections require teardown closure, and uploads require an earlier connection. + Diagnostics: ``missing-ssh-global``, ``missing-ssh-credential``, ``missing-ssh-connect``, ``missing-ssh-close``. +* ``step`` — IndexedStep is runtime-generated and cannot be authored in recipe YAML. + Diagnostics: ``internal-step-type``. +* ``sequence`` — serial_number is accepted with a warning and omitted from the typed model. + Diagnostics: ``legacy-sequence-field``. +* ``step and input mapping`` — Noncanonical step casing and omitted direct input types are normalized with warnings. + Diagnostics: ``noncanonical-step-type``, ``implicit-direct-input``. + +Canonicalization +---------------- + +The parser normalizes step type casing, implicit direct inputs, mapping +defaults, and optional flags. Canonical serialization uses explicit YAML +document starts and stable field ordering. Comments and original formatting +are not preserved; parse/dump/reparse model equality is the guarantee. + +``IndexedStep`` is reserved for runtime construction and cannot be authored +as a recipe step. diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index bddfeb3..32e176b 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -7,8 +7,9 @@ Recipe Language Architecture This page describes the recipe language model, the isolated parser, and the rules for evolving them. The parser is currently independent of recipe -execution, ``verify_recipe``, YamVIEW, and Sphinx reference generation. Those -consumers will move onto the shared model in later integration work. +execution, ``verify_recipe``, YamVIEW, and Sphinx reference publication. Those +runtime and UI consumers will move onto the shared model in later integration +work. Design goals ------------ @@ -31,6 +32,11 @@ The current architecture has two source modules: contract, and constructs immutable typed definitions. It also serializes a typed recipe into canonical YAML. +``pypts.recipe_reference`` + Validates the registered examples and generates the deterministic standalone + RST language reference. The generated artifact remains outside the Sphinx + source tree until framework integration is accepted. + The intended flow is:: recipe YAML @@ -48,7 +54,7 @@ The intended flow is:: | +---- dump_recipe() -> canonical recipe YAML | - +---- future runtime, GUI, and generated-reference consumers + +---- generated reference, and future runtime and GUI consumers Parser operation ---------------- @@ -134,9 +140,10 @@ Keep the dependency direction narrow: runtime, concrete steps, YamVIEW, or Sphinx. * Runtime and UI adapters may consume parser models after integration; parser models must not consume those adapters. -* Syntax reference pages and examples should eventually be generated or - checked from the language specifications. Architecture prose should explain - responsibilities and extension workflows, not duplicate field tables. +* Syntax reference fields, constraints, and examples are generated and checked + from the language specifications. Architecture prose explains + responsibilities and extension workflows rather than duplicating field + tables. These rules keep syntax inspection safe and make the parser usable by command line tools, editors, the GUI, tests, and documentation without constructing @@ -158,8 +165,10 @@ Adding or changing a step specification order and parse/dump/reparse preserves the model. #. During framework integration, update only the adapter that constructs the runtime step from ``StepDefinition``. -#. Regenerate the syntax reference and validate its example once reference - generation is available. +#. Regenerate and check the standalone syntax reference:: + + python -m pypts.recipe_reference docs/generated/recipe_language_reference.rst + python -m pypts.recipe_reference --check docs/generated/recipe_language_reference.rst Changing input or output mappings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -207,7 +216,7 @@ parse, dump, reparse, and compare equal as a model. The comment-only draft is intentionally invalid. Isolation tests also protect the parser from importing runtime, step, GUI, or Sphinx modules. -After the integration and generated-reference phases, verification must also -cover runtime construction equivalence, GUI-produced canonical YAML, registry -and reference completeness, successful Sphinx builds with warnings treated as -errors, and the absence of duplicated consumer-side field or type registries. +After the integration phase, verification must also cover runtime construction +equivalence, GUI-produced canonical YAML, successful Sphinx builds with +warnings treated as errors, and the absence of duplicated consumer-side field +or type registries. diff --git a/src/pypts/recipe_language.py b/src/pypts/recipe_language.py index acc58f9..a418b85 100644 --- a/src/pypts/recipe_language.py +++ b/src/pypts/recipe_language.py @@ -15,6 +15,7 @@ CANONICAL_RECIPE_VERSION = "1.0.0" +NO_DEFAULT = object() @dataclass(frozen=True) @@ -69,6 +70,52 @@ class FieldSpec: value_type: type | tuple[type, ...] | None = None required: bool = False description: str = "" + default: Any = NO_DEFAULT + choices: tuple[Any, ...] = () + legacy: bool = False + choice_diagnostic: str = "invalid-field-value" + + @property + def has_default(self) -> bool: + return self.default is not NO_DEFAULT + + +@dataclass(frozen=True) +class StructureSpec: + """Declarative fields for a recipe document or nested structure.""" + + name: str + description: str + fields: tuple[FieldSpec, ...] + + @property + def fields_by_name(self) -> dict[str, FieldSpec]: + return {field.name: field for field in self.fields} + + +@dataclass(frozen=True) +class MappingSpec: + """Declarative contract for one input or output mapping variant.""" + + name: str + description: str + fields: tuple[FieldSpec, ...] + example: Mapping[str, Any] + verdict: bool = False + + @property + def fields_by_name(self) -> dict[str, FieldSpec]: + return {field.name: field for field in self.fields} + + +@dataclass(frozen=True) +class ConstraintSpec: + """Human-readable metadata for a semantic language constraint.""" + + code: str + scope: str + description: str + diagnostic_codes: tuple[str, ...] = () @dataclass(frozen=True) @@ -92,11 +139,17 @@ def fields_by_name(self) -> dict[str, FieldSpec]: FieldSpec("step_name", str, required=True, description="Human-readable step name."), FieldSpec("description", str, required=True, description="Purpose of the step."), FieldSpec("id", str, description="Optional stable step identifier."), - FieldSpec("skip", bool, description="Skip execution; defaults to false."), - FieldSpec("critical", bool, description="Stop on error when policy permits continuation."), - FieldSpec("continue_on_error", bool, description="Per-step error policy."), - FieldSpec("input_mapping", dict, description="Named input sources."), - FieldSpec("output_mapping", dict, description="Named verdicts and destinations."), + FieldSpec("skip", bool, description="Skip execution.", default=False), + FieldSpec("critical", bool, description="Stop on error when policy permits continuation.", default=False), + FieldSpec("continue_on_error", bool, description="Per-step error policy.", default=False), + FieldSpec("input_mapping", dict, description="Named input sources.", default={}), + FieldSpec("output_mapping", dict, description="Named verdicts and destinations.", default={}), +) + +COMMON_STEP_SPEC = StructureSpec( + "step", + "Fields accepted by every authorable recipe step.", + COMMON_STEP_FIELDS, ) @@ -114,15 +167,20 @@ def _step_spec( STEP_SPECS = ( _step_spec( "PythonModuleStep", - FieldSpec("action_type", str, required=True), - FieldSpec("module", str, required=True), - FieldSpec("method_name", str), + FieldSpec( + "action_type", str, required=True, + description="Operation performed on the Python module.", + choices=("method", "read_attribute", "write_attribute"), + choice_diagnostic="invalid-action-type", + ), + FieldSpec("module", str, required=True, description="Python module path."), + FieldSpec("method_name", str, description="Method name required by method actions."), example={"steptype": "PythonModuleStep", "step_name": "Run test", "description": "Run a Python test method.", "action_type": "method", "module": "tests.py", "method_name": "run", "input_mapping": {}, "output_mapping": {}}, description="Calls a method or reads/writes an attribute in a Python module.", ), _step_spec( "SequenceStep", - FieldSpec("sequence", dict, required=True), + FieldSpec("sequence", dict, required=True, description="Internal sequence reference."), example={"steptype": "SequenceStep", "step_name": "Run calibration", "description": "Run an internal sequence.", "sequence": {"type": "internal", "name": "Calibration"}, "input_mapping": {}, "output_mapping": {}}, description="Runs another sequence as a step.", ), @@ -139,16 +197,16 @@ def _step_spec( ), _step_spec( "UserLoadingStep", - FieldSpec("file_save_location", dict), + FieldSpec("file_save_location", dict, description="Local or global destination for the selected file."), example={"steptype": "UserLoadingStep", "step_name": "Load configuration", "description": "Ask the operator for a file.", "input_mapping": {"message": {"type": "direct", "value": "Choose a file"}}, "output_mapping": {"output": {"type": "passfail"}}}, description="Prompts the operator to select a file.", ), _step_spec( "UserRunMethodStep", - FieldSpec("trigger_response", (str, list, dict)), - FieldSpec("action_type", str), - FieldSpec("module", str), - FieldSpec("method_name", str), + FieldSpec("trigger_response", (str, list, dict), description="Operator response that triggers execution."), + FieldSpec("action_type", str, description="Optional Python action type."), + FieldSpec("module", str, description="Optional Python module path."), + FieldSpec("method_name", str, description="Optional Python method name."), example={"steptype": "UserRunMethodStep", "step_name": "Run calibration", "description": "Run on operator confirmation.", "trigger_response": "run", "action_type": "method", "module": "tests.py", "method_name": "calibrate", "input_mapping": {}, "output_mapping": {"output": {"type": "passfail"}}}, description="Optionally runs a Python method after an operator response.", ), @@ -174,10 +232,10 @@ def _step_spec( ), _step_spec( "SSHUploadStep", - FieldSpec("files", list, required=True), - FieldSpec("permissions", (int, str)), - FieldSpec("skip_if_sha256_match", bool), - FieldSpec("local_package", str), + FieldSpec("files", list, required=True, description="Local and remote file pairs to upload."), + FieldSpec("permissions", (int, str), description="Optional remote permissions."), + FieldSpec("skip_if_sha256_match", bool, description="Skip files whose remote checksum matches.", default=False), + FieldSpec("local_package", str, description="Optional package containing local resources."), example={"steptype": "SSHUploadStep", "step_name": "Deploy", "description": "Upload a file to the target.", "files": [{"local": "bin/tool", "remote": "/tmp/tool"}], "output_mapping": {"passed": {"type": "passfail"}}}, description="Uploads files through an SSH connection.", ), @@ -192,18 +250,242 @@ def _step_spec( STEP_SPECS_BY_NAME = {spec.name.casefold(): spec for spec in STEP_SPECS} HEADER_FIELDS = ( - FieldSpec("name", str, required=True), FieldSpec("version", str, required=True), - FieldSpec("recipe_version", str, required=True), FieldSpec("description", str, required=True), - FieldSpec("main_sequence", str, required=True), FieldSpec("globals", dict, required=True), - FieldSpec("continue_on_error", bool), FieldSpec("report", str), - FieldSpec("report_name_include_serial", bool), FieldSpec("test_package", str), + FieldSpec("name", str, required=True, description="Human-readable recipe name."), + FieldSpec("version", str, required=True, description="Version of this recipe."), + FieldSpec( + "recipe_version", str, required=True, + description="Version of the recipe language contract.", + choices=(CANONICAL_RECIPE_VERSION,), + choice_diagnostic="unsupported-recipe-version", + ), + FieldSpec("description", str, required=True, description="Purpose of the recipe."), + FieldSpec("main_sequence", str, required=True, description="Sequence where execution begins."), + FieldSpec("globals", dict, required=True, description="Recipe-wide variables."), + FieldSpec("continue_on_error", bool, description="Recipe-wide error policy.", default=None), + FieldSpec( + "report", str, description="Report file mode.", default="overwrite", + choices=("overwrite", "append"), choice_diagnostic="invalid-report-mode", + ), + FieldSpec( + "report_name_include_serial", bool, + description="Include the serial number in the report name.", default=False, + ), + FieldSpec("test_package", str, description="Package containing recipe test modules.", default=None), +) +HEADER_SPEC = StructureSpec( + "recipe header", + "The first YAML document; identifies the recipe and its entry sequence.", + HEADER_FIELDS, ) + SEQUENCE_FIELDS = ( - FieldSpec("sequence_name", str, required=True), FieldSpec("description", str, required=True), - FieldSpec("parameters", dict, required=True), FieldSpec("outputs", dict, required=True), - FieldSpec("locals", dict, required=True), FieldSpec("setup_steps", list, required=True), - FieldSpec("steps", list, required=True), FieldSpec("teardown_steps", list, required=True), - FieldSpec("serial_number", (str, int), description="Legacy runtime-ignored sequence metadata."), + FieldSpec("sequence_name", str, required=True, description="Unique sequence name."), + FieldSpec("description", str, required=True, description="Purpose of the sequence."), + FieldSpec("parameters", dict, required=True, description="Reserved sequence input metadata."), + FieldSpec("outputs", dict, required=True, description="Reserved sequence output metadata."), + FieldSpec("locals", dict, required=True, description="Variables local to the sequence."), + FieldSpec("setup_steps", list, required=True, description="Steps run before the main steps."), + FieldSpec("steps", list, required=True, description="Ordered main steps."), + FieldSpec("teardown_steps", list, required=True, description="Steps run during teardown."), + FieldSpec( + "serial_number", (str, int), + description="Runtime-ignored legacy sequence metadata.", legacy=True, + ), +) +SEQUENCE_SPEC = StructureSpec( + "sequence", + "Each YAML document after the header defines one named sequence.", + SEQUENCE_FIELDS, +) + +SEQUENCE_REFERENCE_SPEC = StructureSpec( + "internal sequence reference", + "Reference used by SequenceStep.", + ( + FieldSpec("type", str, required=True, description="Reference kind.", choices=("internal",)), + FieldSpec("name", str, required=True, description="Target sequence name."), + ), +) + +FILE_SAVE_LOCATION_SPEC = StructureSpec( + "file save location", + "Destination used by UserLoadingStep.", + ( + FieldSpec("type", str, required=True, description="Variable scope.", choices=("local", "global")), + FieldSpec("variable", str, required=True, description="Destination variable name."), + ), +) + +INPUT_MAPPING_SPECS = ( + MappingSpec( + "direct", "Provides a literal value.", + ( + FieldSpec("type", str, description="Input source type.", default="direct", choices=("direct",)), + FieldSpec("value", required=True, description="Literal input value."), + FieldSpec("indexed", bool, description="Expand a list into indexed steps.", default=False), + ), + {"type": "direct", "value": 1}, + ), + MappingSpec( + "local", "Reads a sequence-local variable.", + ( + FieldSpec("type", str, required=True, description="Input source type.", choices=("local",)), + FieldSpec("local_name", str, required=True, description="Local variable name."), + FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), + ), + {"type": "local", "local_name": "local_value"}, + ), + MappingSpec( + "global", "Reads a recipe-global variable.", + ( + FieldSpec("type", str, required=True, description="Input source type.", choices=("global",)), + FieldSpec("global_name", str, required=True, description="Global variable name."), + FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), + ), + {"type": "global", "global_name": "global_value"}, + ), + MappingSpec( + "method", "Resolves a method reference for the step.", + ( + FieldSpec("type", str, required=True, description="Input source type.", choices=("method",)), + FieldSpec("value", required=True, description="Method reference."), + FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), + ), + {"type": "method", "value": "helper"}, + ), +) +INPUT_MAPPING_SPECS_BY_NAME = {spec.name: spec for spec in INPUT_MAPPING_SPECS} + +OUTPUT_MAPPING_SPECS = ( + MappingSpec( + "passfail", "Interprets the output as a pass/fail verdict.", + (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("passfail",)),), + {"type": "passfail"}, verdict=True, + ), + MappingSpec( + "equals", "Passes when the output equals the configured value.", + ( + FieldSpec("type", str, required=True, description="Output mapping type.", choices=("equals",)), + FieldSpec("value", required=True, description="Expected value."), + ), + {"type": "equals", "value": 3}, verdict=True, + ), + MappingSpec( + "range", "Passes when the output is within an inclusive range.", + ( + FieldSpec("type", str, required=True, description="Output mapping type.", choices=("range",)), + FieldSpec("min", required=True, description="Minimum accepted value."), + FieldSpec("max", required=True, description="Maximum accepted value."), + ), + {"type": "range", "min": 1, "max": 4}, verdict=True, + ), + MappingSpec( + "passthrough", "Uses the nested result without adding a verdict.", + (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("passthrough",)),), + {"type": "passthrough"}, verdict=True, + ), + MappingSpec( + "local", "Stores the output in a sequence-local variable.", + ( + FieldSpec("type", str, required=True, description="Output mapping type.", choices=("local",)), + FieldSpec("local_name", str, required=True, description="Local destination variable."), + ), + {"type": "local", "local_name": "saved"}, + ), + MappingSpec( + "global", "Stores the output in a recipe-global variable.", + ( + FieldSpec("type", str, required=True, description="Output mapping type.", choices=("global",)), + FieldSpec("global_name", str, required=True, description="Global destination variable."), + ), + {"type": "global", "global_name": "saved"}, + ), + MappingSpec( + "image", "Publishes an image output for presentation.", + (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("image",)),), + {"type": "image"}, + ), +) +OUTPUT_MAPPING_SPECS_BY_NAME = {spec.name: spec for spec in OUTPUT_MAPPING_SPECS} + +CONSTRAINT_SPECS = ( + ConstraintSpec( + "safe-source", "parser", + "Sources must be readable text and safe, non-recursive YAML without duplicate keys.", + ("invalid-source", "file-read-error", "yaml-syntax-error", "yaml-construction-error", "unsafe-yaml", "recursive-alias", "duplicate-key"), + ), + ConstraintSpec( + "declared-fields", "documents and steps", + "Only declared fields, field types, and allowed values are accepted.", + ("missing-field", "invalid-field-type", "invalid-field-value", "unknown-field", "unsupported-recipe-version", "invalid-report-mode", "invalid-action-type"), + ), + ConstraintSpec( + "mapping-shape", "input and output mappings", + "Mapping variants accept only their declared fields and require their declared values.", + ("invalid-input-mapping", "unknown-input-source", "unknown-input-field", "invalid-input-field-type", "missing-input-source-value", "invalid-output-mapping", "unknown-output-type", "unknown-output-field", "invalid-output-field-type", "missing-output-field"), + ), + ConstraintSpec( + "recipe-documents", "recipe", + "A recipe is safe multi-document YAML with one header followed by at least one sequence.", + ("empty-recipe", "invalid-header", "invalid-sequence"), + ), + ConstraintSpec( + "registered-step", "step", "Every step is a mapping with a registered step type.", + ("invalid-step", "unknown-step-type"), + ), + ConstraintSpec( + "unique-sequences", "sequence", + "Sequence names are unique and main_sequence names an existing sequence.", + ("duplicate-sequence", "unknown-main-sequence"), + ), + ConstraintSpec( + "internal-sequence-reference", "SequenceStep", + "Only internal references are accepted and the named target sequence must exist.", + ("invalid-sequence-reference", "unknown-sequence-reference"), + ), + ConstraintSpec( + "method-action-name", "PythonModuleStep", "A method action requires method_name.", + ("missing-method-name",), + ), + ConstraintSpec( + "file-save-location", "UserLoadingStep", "A file destination names a local or global variable.", + ("invalid-file-save-location",), + ), + ConstraintSpec( + "indexed-inputs", "input mapping", + "Indexed inputs are direct lists and all indexed lists on a step have equal length.", + ("invalid-indexed-flag", "invalid-indexed-input", "unequal-indexed-inputs"), + ), + ConstraintSpec( + "passthrough-verdict", "output mapping", "passthrough must be the only verdict mapping on its step.", + ("mixed-passthrough",), + ), + ConstraintSpec( + "required-step-inputs", "step", "Step-specific required input names must be present.", + ("missing-required-input", "missing-input-mapping"), + ), + ConstraintSpec( + "ssh-context", "SSH steps", + "SSH steps require connection globals; setup connections require teardown closure, and uploads require an earlier connection.", + ("missing-ssh-global", "missing-ssh-credential", "missing-ssh-connect", "missing-ssh-close"), + ), + ConstraintSpec( + "internal-step", "step", "IndexedStep is runtime-generated and cannot be authored in recipe YAML.", + ("internal-step-type",), + ), + ConstraintSpec( + "legacy-sequence-field", "sequence", "serial_number is accepted with a warning and omitted from the typed model.", + ("legacy-sequence-field",), + ), + ConstraintSpec( + "canonical-spelling", "step and input mapping", + "Noncanonical step casing and omitted direct input types are normalized with warnings.", + ("noncanonical-step-type", "implicit-direct-input"), + ), +) +CONSTRAINT_SPECS_BY_CODE = {spec.code: spec for spec in CONSTRAINT_SPECS} +DOCUMENTED_DIAGNOSTIC_CODES = frozenset( + code for spec in CONSTRAINT_SPECS for code in spec.diagnostic_codes ) @@ -217,13 +499,26 @@ def _type_name(value_type: type | tuple[type, ...]) -> str: return " or ".join(value.__name__ for value in values) +def _matches_type(value: Any, value_type: type | tuple[type, ...]) -> bool: + if value_type is bool: + return type(value) is bool + return isinstance(value, value_type) + + def _check_fields(value: Mapping[str, Any], fields: Iterable[FieldSpec], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: specs = {spec.name: spec for spec in fields} for name, spec in specs.items(): if spec.required and name not in value: diagnostics.append(Diagnostic("missing-field", f"Missing required field '{name}'.", path + (name,))) - elif name in value and spec.value_type is not None and (type(value[name]) is not bool if spec.value_type is bool else not isinstance(value[name], spec.value_type)): + elif name in value and spec.value_type is not None and not _matches_type(value[name], spec.value_type): diagnostics.append(Diagnostic("invalid-field-type", f"Field '{name}' must be {_type_name(spec.value_type)}.", path + (name,))) + elif name in value and spec.choices and value[name] not in spec.choices: + choices = ", ".join(repr(choice) for choice in spec.choices) + diagnostics.append(Diagnostic( + spec.choice_diagnostic, + f"Field '{name}' must be one of: {choices}.", + path + (name,), + )) for name in value: if name not in specs: diagnostics.append(Diagnostic("unknown-field", f"Unknown field '{name}'.", path + (name,))) @@ -237,21 +532,26 @@ def _validate_input_mappings(mapping: Mapping[str, Any], path: tuple[str | int, diagnostics.append(Diagnostic("invalid-input-mapping", "Input mapping must be a dictionary.", item_path)) continue source = config.get("type", "direct") - if source not in {"direct", "local", "global", "method"}: + if source not in INPUT_MAPPING_SPECS_BY_NAME: diagnostics.append(Diagnostic("unknown-input-source", f"Unknown input source '{source}'.", item_path + ("type",))) continue - allowed = { - "direct": {"type", "value", "indexed"}, - "local": {"type", "local_name", "indexed"}, - "global": {"type", "global_name", "indexed"}, - "method": {"type", "value", "indexed"}, - }[source] + spec = INPUT_MAPPING_SPECS_BY_NAME[source] + allowed = spec.fields_by_name for field_name in config: if field_name not in allowed: diagnostics.append(Diagnostic("unknown-input-field", f"Unknown field '{field_name}' for input source '{source}'.", item_path + (field_name,))) - required_key = {"direct": "value", "local": "local_name", "global": "global_name", "method": "value"}[source] - if required_key not in config: - diagnostics.append(Diagnostic("missing-input-source-value", f"Input source '{source}' requires '{required_key}'.", item_path)) + for field_name, field_spec in allowed.items(): + if field_name in {"type", "indexed"} or field_name not in config or field_spec.value_type is None: + continue + if not _matches_type(config[field_name], field_spec.value_type): + diagnostics.append(Diagnostic( + "invalid-input-field-type", + f"Input field '{field_name}' must be {_type_name(field_spec.value_type)}.", + item_path + (field_name,), + )) + for field_spec in spec.fields: + if field_spec.required and field_spec.name not in config: + diagnostics.append(Diagnostic("missing-input-source-value", f"Input source '{source}' requires '{field_spec.name}'.", item_path)) if "indexed" in config and type(config["indexed"]) is not bool: diagnostics.append(Diagnostic("invalid-indexed-flag", "'indexed' must be boolean.", item_path + ("indexed",))) if config.get("indexed"): @@ -265,32 +565,34 @@ def _validate_input_mappings(mapping: Mapping[str, Any], path: tuple[str | int, def _validate_output_mappings(mapping: Mapping[str, Any], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: verdicts: list[str] = [] - requirements = {"equals": "value", "range": ("min", "max"), "local": "local_name", "global": "global_name"} for name, config in mapping.items(): item_path = path + (name,) if not isinstance(config, Mapping) or not isinstance(config.get("type"), str): diagnostics.append(Diagnostic("invalid-output-mapping", "Output mapping requires a string 'type'.", item_path)) continue kind = config["type"] - if kind not in {"passfail", "equals", "range", "passthrough", "local", "global", "image"}: + if kind not in OUTPUT_MAPPING_SPECS_BY_NAME: diagnostics.append(Diagnostic("unknown-output-type", f"Unknown output type '{kind}'.", item_path + ("type",))) continue - allowed = { - "passfail": {"type"}, "equals": {"type", "value"}, - "range": {"type", "min", "max"}, "passthrough": {"type"}, - "local": {"type", "local_name"}, "global": {"type", "global_name"}, - "image": {"type"}, - }[kind] + spec = OUTPUT_MAPPING_SPECS_BY_NAME[kind] + allowed = spec.fields_by_name for field_name in config: if field_name not in allowed: diagnostics.append(Diagnostic("unknown-output-field", f"Unknown field '{field_name}' for output type '{kind}'.", item_path + (field_name,))) - if kind in {"passfail", "equals", "range", "passthrough"}: + for field_name, field_spec in allowed.items(): + if field_name == "type" or field_name not in config or field_spec.value_type is None: + continue + if not _matches_type(config[field_name], field_spec.value_type): + diagnostics.append(Diagnostic( + "invalid-output-field-type", + f"Output field '{field_name}' must be {_type_name(field_spec.value_type)}.", + item_path + (field_name,), + )) + if spec.verdict: verdicts.append(kind) - required = requirements.get(kind, ()) - required = (required,) if isinstance(required, str) else required - for field_name in required: - if field_name not in config: - diagnostics.append(Diagnostic("missing-output-field", f"Output type '{kind}' requires '{field_name}'.", item_path)) + for field_spec in spec.fields: + if field_spec.required and field_spec.name not in config: + diagnostics.append(Diagnostic("missing-output-field", f"Output type '{kind}' requires '{field_spec.name}'.", item_path)) if "passthrough" in verdicts and len(verdicts) != 1: diagnostics.append(Diagnostic("mixed-passthrough", "'passthrough' must be the sole verdict mapping.", path)) @@ -319,8 +621,6 @@ def _validate_step(step: Any, path: tuple[str | int, ...], diagnostics: list[Dia if "output_mapping" in step and isinstance(step["output_mapping"], Mapping): _validate_output_mappings(step["output_mapping"], path + ("output_mapping",), diagnostics) if canonical_name == "PythonModuleStep": - if step.get("action_type") not in {"method", "read_attribute", "write_attribute"}: - diagnostics.append(Diagnostic("invalid-action-type", "PythonModuleStep action_type must be method, read_attribute, or write_attribute.", path + ("action_type",))) if step.get("action_type") == "method" and not step.get("method_name"): diagnostics.append(Diagnostic("missing-method-name", "PythonModuleStep method action requires method_name.", path + ("method_name",))) if canonical_name == "SequenceStep": @@ -349,10 +649,6 @@ def validate_recipe_documents(documents: Iterable[Any]) -> ValidationResult: if not isinstance(header, Mapping): return ValidationResult((Diagnostic("invalid-header", "The first document must be the recipe header.", (0,)),)) _check_fields(header, HEADER_FIELDS, (0,), diagnostics) - if header.get("recipe_version") != CANONICAL_RECIPE_VERSION: - diagnostics.append(Diagnostic("unsupported-recipe-version", f"recipe_version must be '{CANONICAL_RECIPE_VERSION}'.", (0, "recipe_version"))) - if "report" in header and header.get("report") not in {"overwrite", "append"}: - diagnostics.append(Diagnostic("invalid-report-mode", "report must be 'overwrite' or 'append'.", (0, "report"))) sequences: dict[str, Mapping[str, Any]] = {} sequence_step_types: dict[str, dict[str, list[str]]] = {} for doc_index, sequence in enumerate(docs[1:], start=1): diff --git a/src/pypts/recipe_reference.py b/src/pypts/recipe_reference.py new file mode 100644 index 0000000..21e28e8 --- /dev/null +++ b/src/pypts/recipe_reference.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Generate the standalone reference for the pypts recipe language.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +from pathlib import Path +import sys +from typing import Any + +import yaml + +from pypts.recipe_language import ( + CANONICAL_RECIPE_VERSION, + COMMON_STEP_FIELDS, + CONSTRAINT_SPECS, + FieldSpec, + HEADER_SPEC, + INPUT_MAPPING_SPECS, + MappingSpec, + OUTPUT_MAPPING_SPECS, + SEQUENCE_SPEC, + STEP_SPECS, + StepSpec, +) +from pypts.recipe_parser import dump_recipe, parse_recipe_text + + +DEFAULT_REFERENCE_PATH = Path("docs/generated/recipe_language_reference.rst") + + +def _header() -> dict[str, Any]: + return { + "name": "Recipe language reference fixture", + "version": "1.0", + "recipe_version": CANONICAL_RECIPE_VERSION, + "description": "Executable generated-reference fixture.", + "main_sequence": "Main", + "globals": { + "ssh_client": None, + "host": "target", + "user": "root", + "port": 22, + "password": "secret", + }, + } + + +def _sequence(name: str, steps: list[Mapping[str, Any]] | None = None) -> dict[str, Any]: + return { + "sequence_name": name, + "description": f"{name} sequence.", + "parameters": {}, + "outputs": {}, + "locals": {"local_value": 1}, + "setup_steps": [], + "steps": list(steps or []), + "teardown_steps": [], + } + + +def _documents_with_step(step: Mapping[str, Any]) -> list[dict[str, Any]]: + authored = dict(step) + main = _sequence("Main", [authored]) + documents = [_header(), main] + if authored.get("steptype") == "SequenceStep": + target = authored.get("sequence", {}).get("name") + if isinstance(target, str) and target != "Main": + documents.append(_sequence(target)) + if authored.get("steptype") == "SSHUploadStep": + main["setup_steps"] = [{ + "steptype": "SSHConnectStep", + "step_name": "Connect", + "description": "Open the fixture connection.", + }] + main["teardown_steps"] = [{ + "steptype": "SSHCloseStep", + "step_name": "Close", + "description": "Close the fixture connection.", + }] + return documents + + +def _fixture_text(step: Mapping[str, Any]) -> str: + return yaml.safe_dump_all( + _documents_with_step(step), + explicit_start=True, + sort_keys=False, + allow_unicode=True, + ) + + +def _canonical_step_example(spec: StepSpec) -> Mapping[str, Any]: + result = parse_recipe_text(_fixture_text(spec.example), f"reference:{spec.name}") + if result.diagnostics: + details = "; ".join(f"{item.code}: {item.message}" for item in result.diagnostics) + raise ValueError(f"Invalid canonical example for {spec.name}: {details}") + documents = list(yaml.safe_load_all(dump_recipe(result.require_recipe()))) + main = next(document for document in documents[1:] if document["sequence_name"] == "Main") + return main["steps"][0] + + +def _validate_mapping_example(spec: MappingSpec, *, output: bool) -> None: + step: dict[str, Any] = { + "steptype": "PythonModuleStep", + "step_name": f"{spec.name} mapping", + "description": "Executable mapping fixture.", + "action_type": "method", + "module": "tests.py", + "method_name": "run", + "input_mapping": {}, + "output_mapping": {}, + } + mapping_name = "output_mapping" if output else "input_mapping" + step[mapping_name] = {"example": dict(spec.example)} + result = parse_recipe_text(_fixture_text(step), f"reference:{mapping_name}:{spec.name}") + if result.diagnostics: + details = "; ".join(f"{item.code}: {item.message}" for item in result.diagnostics) + raise ValueError(f"Invalid {mapping_name} example for {spec.name}: {details}") + + +def _type_name(field: FieldSpec) -> str: + if field.value_type is None: + return "any" + values = field.value_type if isinstance(field.value_type, tuple) else (field.value_type,) + return " or ".join(value.__name__ for value in values) + + +def _literal(value: Any) -> str: + if value == {}: + return "{}" + if value is None: + return "null" + if value is True: + return "true" + if value is False: + return "false" + return str(value) + + +def _requirement(field: FieldSpec) -> str: + parts = ["required" if field.required else "optional"] + if field.has_default: + parts.append(f"default: ``{_literal(field.default)}``") + if field.choices: + choices = ", ".join(f"``{_literal(value)}``" for value in field.choices) + parts.append(f"allowed: {choices}") + if field.legacy: + parts.append("legacy") + return "; ".join(parts) + + +def _field_table(fields: Sequence[FieldSpec]) -> list[str]: + lines = [ + ".. list-table::", + " :header-rows: 1", + " :widths: 18 14 28 40", + "", + " * - Field", + " - Type", + " - Requirement", + " - Description", + ] + for field in fields: + lines.extend(( + f" * - ``{field.name}``", + f" - {_type_name(field)}", + f" - {_requirement(field)}", + f" - {field.description}", + )) + return lines + + +def _yaml_block(value: Mapping[str, Any]) -> list[str]: + rendered = yaml.safe_dump( + dict(value), sort_keys=False, default_flow_style=False, allow_unicode=True, + ).rstrip() + return [".. code-block:: yaml", ""] + [f" {line}" for line in rendered.splitlines()] + + +def _mapping_section(title: str, specs: Sequence[MappingSpec], prefix: str) -> list[str]: + lines = [title, "-" * len(title), ""] + for spec in specs: + lines.extend(( + f".. _recipe-{prefix}-{spec.name}:", + "", + f"**``{spec.name}``**", + "", + spec.description, + "", + )) + lines.extend(_field_table(spec.fields)) + lines.extend(("", "Canonical mapping:", "")) + lines.extend(_yaml_block(spec.example)) + lines.append("") + return lines + + +def render_recipe_reference() -> str: + """Render the deterministic standalone recipe-language RST reference.""" + public_steps = tuple(spec for spec in STEP_SPECS if spec.source_allowed) + examples = {spec.name: _canonical_step_example(spec) for spec in public_steps} + for spec in INPUT_MAPPING_SPECS: + _validate_mapping_example(spec, output=False) + for spec in OUTPUT_MAPPING_SPECS: + _validate_mapping_example(spec, output=True) + + lines = [ + ".. SPDX-FileCopyrightText: 2026 CERN ", + "..", + ".. SPDX-License-Identifier: CC-BY-SA-4.0", + "..", + ".. This file is generated by pypts.recipe_reference. Do not edit it manually.", + "", + "Recipe Language Reference", + "=========================", + "", + f"Canonical recipe language version: ``{CANONICAL_RECIPE_VERSION}``.", + "", + "Document grammar", + "----------------", + "", + "A recipe is safe multi-document YAML. The first document is one recipe", + "header and every following document is one sequence. At least one sequence", + "is required, and ``main_sequence`` must name one of them.", + "", + "Recipe header", + "-------------", + "", + HEADER_SPEC.description, + "", + ] + lines.extend(_field_table(HEADER_SPEC.fields)) + lines.extend(("", "Sequence", "--------", "", SEQUENCE_SPEC.description, "")) + lines.extend(_field_table(SEQUENCE_SPEC.fields)) + lines.extend(( + "", + "Common step fields", + "------------------", + "", + "These fields are shared by every authorable step type.", + "", + )) + lines.extend(_field_table(COMMON_STEP_FIELDS)) + lines.extend(("", "Registered step types", "---------------------", "")) + common_names = {field.name for field in COMMON_STEP_FIELDS} + for spec in public_steps: + lines.extend(( + f".. _recipe-step-{spec.name.lower()}:", + "", + spec.name, + "~" * len(spec.name), + "", + spec.description, + "", + )) + specific_fields = tuple(field for field in spec.fields if field.name not in common_names) + if specific_fields: + lines.extend(_field_table(specific_fields)) + lines.append("") + if spec.required_inputs: + required = ", ".join(f"``{name}``" for name in spec.required_inputs) + lines.extend((f"Required input names: {required}.", "")) + lines.extend(("Canonical example:", "")) + lines.extend(_yaml_block(examples[spec.name])) + lines.append("") + lines.extend(_mapping_section("Input mapping types", INPUT_MAPPING_SPECS, "input")) + lines.extend(_mapping_section("Output mapping types", OUTPUT_MAPPING_SPECS, "output")) + lines.extend(("Semantic constraints", "--------------------", "")) + for constraint in CONSTRAINT_SPECS: + diagnostics = ", ".join(f"``{code}``" for code in constraint.diagnostic_codes) + lines.extend(( + f"* ``{constraint.scope}`` — {constraint.description}", + f" Diagnostics: {diagnostics}.", + )) + lines.extend(( + "", + "Canonicalization", + "----------------", + "", + "The parser normalizes step type casing, implicit direct inputs, mapping", + "defaults, and optional flags. Canonical serialization uses explicit YAML", + "document starts and stable field ordering. Comments and original formatting", + "are not preserved; parse/dump/reparse model equality is the guarantee.", + "", + "``IndexedStep`` is reserved for runtime construction and cannot be authored", + "as a recipe step.", + "", + )) + return "\n".join(lines) + + +def write_recipe_reference(path: str | Path = DEFAULT_REFERENCE_PATH) -> None: + """Write the generated reference, creating its parent directory.""" + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(render_recipe_reference(), encoding="utf-8") + + +def check_recipe_reference(path: str | Path = DEFAULT_REFERENCE_PATH) -> bool: + """Return whether an existing reference exactly matches generated output.""" + destination = Path(path) + try: + current = destination.read_text(encoding="utf-8") + except (OSError, UnicodeError): + return False + return current == render_recipe_reference() + + +def main(argv: Sequence[str] | None = None) -> int: + """Generate or check the standalone reference artifact.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", nargs="?", type=Path, default=DEFAULT_REFERENCE_PATH) + parser.add_argument("--check", action="store_true", help="fail if the artifact is missing or stale") + arguments = parser.parse_args(argv) + if arguments.check: + if check_recipe_reference(arguments.path): + return 0 + print(f"Recipe language reference is missing or stale: {arguments.path}", file=sys.stderr) + return 1 + try: + write_recipe_reference(arguments.path) + except OSError as error: + print(f"Could not write recipe language reference: {error}", file=sys.stderr) + return 2 + print(f"Wrote recipe language reference: {arguments.path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit_tests/test_recipe_reference.py b/tests/unit_tests/test_recipe_reference.py new file mode 100644 index 0000000..2c979bd --- /dev/null +++ b/tests/unit_tests/test_recipe_reference.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later + +import ast +from pathlib import Path + +import pytest +import yaml + +from pypts.recipe_language import ( + COMMON_STEP_SPEC, + CONSTRAINT_SPECS, + DOCUMENTED_DIAGNOSTIC_CODES, + FILE_SAVE_LOCATION_SPEC, + HEADER_SPEC, + INPUT_MAPPING_SPECS, + OUTPUT_MAPPING_SPECS, + SEQUENCE_REFERENCE_SPEC, + SEQUENCE_SPEC, + STEP_SPECS, +) +from pypts.recipe_parser import ( + DirectInput, + EqualsOutput, + GlobalInput, + GlobalOutput, + ImageOutput, + LocalInput, + LocalOutput, + MethodInput, + PassFailOutput, + PassthroughOutput, + RangeOutput, + parse_recipe_text, +) +from pypts.recipe_reference import ( + _fixture_text, + check_recipe_reference, + main, + render_recipe_reference, +) + + +ROOT = Path(__file__).parents[2] +REFERENCE = ROOT / "docs" / "generated" / "recipe_language_reference.rst" + + +def _all_structures(): + return ( + HEADER_SPEC, + SEQUENCE_SPEC, + COMMON_STEP_SPEC, + SEQUENCE_REFERENCE_SPEC, + FILE_SAVE_LOCATION_SPEC, + ) + + +def test_registry_metadata_is_complete_and_unambiguous(): + step_names = [spec.name.casefold() for spec in STEP_SPECS] + assert len(step_names) == len(set(step_names)) + + for structure in _all_structures(): + assert structure.name and structure.description + field_names = [field.name for field in structure.fields] + assert len(field_names) == len(set(field_names)) + assert all(field.description for field in structure.fields) + + for specs in (INPUT_MAPPING_SPECS, OUTPUT_MAPPING_SPECS): + names = [spec.name for spec in specs] + assert len(names) == len(set(names)) + for spec in specs: + assert spec.description and spec.example + assert spec.example["type"] == spec.name + assert all(field.description for field in spec.fields) + + for spec in STEP_SPECS: + assert spec.name and spec.description and spec.example + assert all(field.description for field in spec.fields) + + +def test_constraint_diagnostic_registry_is_complete_and_unique(): + constraint_codes = [spec.code for spec in CONSTRAINT_SPECS] + diagnostic_codes = [code for spec in CONSTRAINT_SPECS for code in spec.diagnostic_codes] + assert len(constraint_codes) == len(set(constraint_codes)) + assert len(diagnostic_codes) == len(set(diagnostic_codes)) + assert set(diagnostic_codes) == DOCUMENTED_DIAGNOSTIC_CODES + assert all(spec.scope and spec.description and spec.diagnostic_codes for spec in CONSTRAINT_SPECS) + + discovered = set() + for relative in ("src/pypts/recipe_language.py", "src/pypts/recipe_parser.py"): + tree = ast.parse((ROOT / relative).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not node.args: + continue + function = node.func.id if isinstance(node.func, ast.Name) else None + if function not in {"Diagnostic", "_source_diagnostic"}: + continue + if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): + discovered.add(node.args[0].value) + assert discovered <= DOCUMENTED_DIAGNOSTIC_CODES + + +def test_reference_generator_has_no_runtime_gui_or_sphinx_imports(): + tree = ast.parse((ROOT / "src/pypts/recipe_reference.py").read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} + assert not any( + name == item or name.startswith(item + ".") + for name in imported + for item in forbidden + ) + + +@pytest.mark.parametrize( + "spec", + [spec for spec in STEP_SPECS if spec.source_allowed], + ids=lambda spec: spec.name, +) +def test_every_public_step_example_is_an_executable_parser_fixture(spec): + result = parse_recipe_text(_fixture_text(spec.example), f"fixture:{spec.name}") + assert result.is_valid, result.errors + assert not result.warnings + assert result.require_recipe().sequences[0].steps[0].steptype == spec.name + + +INPUT_TYPES = { + "direct": DirectInput, + "local": LocalInput, + "global": GlobalInput, + "method": MethodInput, +} + + +@pytest.mark.parametrize("spec", INPUT_MAPPING_SPECS, ids=lambda spec: spec.name) +def test_every_input_mapping_example_builds_its_typed_model(spec): + step = { + "steptype": "PythonModuleStep", + "step_name": "Input fixture", + "description": "Input mapping fixture.", + "action_type": "method", + "module": "tests.py", + "method_name": "run", + "input_mapping": {"example": dict(spec.example)}, + "output_mapping": {}, + } + result = parse_recipe_text(_fixture_text(step)) + assert result.is_valid, result.errors + value = result.require_recipe().sequences[0].steps[0].input_mapping["example"] + assert isinstance(value, INPUT_TYPES[spec.name]) + + +OUTPUT_TYPES = { + "passfail": PassFailOutput, + "equals": EqualsOutput, + "range": RangeOutput, + "passthrough": PassthroughOutput, + "local": LocalOutput, + "global": GlobalOutput, + "image": ImageOutput, +} + + +@pytest.mark.parametrize("spec", OUTPUT_MAPPING_SPECS, ids=lambda spec: spec.name) +def test_every_output_mapping_example_builds_its_typed_model(spec): + step = { + "steptype": "PythonModuleStep", + "step_name": "Output fixture", + "description": "Output mapping fixture.", + "action_type": "method", + "module": "tests.py", + "method_name": "run", + "input_mapping": {}, + "output_mapping": {"example": dict(spec.example)}, + } + result = parse_recipe_text(_fixture_text(step)) + assert result.is_valid, result.errors + value = result.require_recipe().sequences[0].steps[0].output_mapping["example"] + assert isinstance(value, OUTPUT_TYPES[spec.name]) + + +def test_mapping_field_types_are_enforced_from_the_registry(): + step = { + "steptype": "PythonModuleStep", + "step_name": "Invalid mapping fields", + "description": "Mapping field type fixture.", + "action_type": "method", + "module": "tests.py", + "method_name": "run", + "input_mapping": {"source": {"type": "local", "local_name": 1}}, + "output_mapping": {"destination": {"type": "global", "global_name": 2}}, + } + result = parse_recipe_text(_fixture_text(step)) + assert {item.code for item in result.errors} == { + "invalid-input-field-type", + "invalid-output-field-type", + } + + +@pytest.mark.parametrize("report", ("overwrite", "append")) +def test_every_report_mode_from_the_registry_parses(report): + step = { + "steptype": "WaitStep", + "step_name": "Wait", + "description": "Report mode fixture.", + "input_mapping": {"wait_time": {"type": "direct", "value": 0}}, + "output_mapping": {}, + } + documents = list(yaml.safe_load_all(_fixture_text(step))) + documents[0]["report"] = report + source = yaml.safe_dump_all(documents, explicit_start=True, sort_keys=False) + assert parse_recipe_text(source).is_valid + + +@pytest.mark.parametrize("action_type", ("method", "read_attribute", "write_attribute")) +def test_every_python_action_type_from_the_registry_parses(action_type): + step = { + "steptype": "PythonModuleStep", + "step_name": "Action fixture", + "description": "Action choice fixture.", + "action_type": action_type, + "module": "tests.py", + "input_mapping": {}, + "output_mapping": {}, + } + if action_type == "method": + step["method_name"] = "run" + assert parse_recipe_text(_fixture_text(step)).is_valid + + +def test_generated_reference_is_complete_and_current(): + rendered = render_recipe_reference() + assert rendered == REFERENCE.read_text(encoding="utf-8") + for spec in STEP_SPECS: + anchor = f".. _recipe-step-{spec.name.lower()}:" + assert rendered.count(anchor) == int(spec.source_allowed) + for spec in INPUT_MAPPING_SPECS: + assert rendered.count(f".. _recipe-input-{spec.name}:") == 1 + for spec in OUTPUT_MAPPING_SPECS: + assert rendered.count(f".. _recipe-output-{spec.name}:") == 1 + + +def test_reference_cli_writes_checks_and_detects_stale_files(tmp_path, capsys): + path = tmp_path / "nested" / "reference.rst" + assert main([str(path)]) == 0 + assert check_recipe_reference(path) + assert main(["--check", str(path)]) == 0 + + path.write_text("stale\n", encoding="utf-8") + assert main(["--check", str(path)]) == 1 + assert path.read_text(encoding="utf-8") == "stale\n" + assert "missing or stale" in capsys.readouterr().err + + missing = tmp_path / "missing.rst" + assert main(["--check", str(missing)]) == 1 + assert not missing.exists() From dba8089bd4b993a74daa7de9255c6cae73c509c6 Mon Sep 17 00:00:00 2001 From: alvaro Date: Thu, 13 Aug 2026 15:18:50 +0200 Subject: [PATCH 05/14] Pydantic spike added to be used as source of truth for the recipe language --- pyproject.toml | 1 + spikes/recipe_pydantic/__init__.py | 28 + spikes/recipe_pydantic/models.py | 380 +++++++++++++ spikes/recipe_pydantic/parser.py | 534 ++++++++++++++++++ spikes/recipe_pydantic/reference.py | 128 +++++ .../recipe_pydantic/test_recipe_pydantic.py | 404 +++++++++++++ 6 files changed, 1475 insertions(+) create mode 100644 spikes/recipe_pydantic/__init__.py create mode 100644 spikes/recipe_pydantic/models.py create mode 100644 spikes/recipe_pydantic/parser.py create mode 100644 spikes/recipe_pydantic/reference.py create mode 100644 spikes/recipe_pydantic/test_recipe_pydantic.py diff --git a/pyproject.toml b/pyproject.toml index e2a0039..4e3c72b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ dev = [ # Include the "test" and "doc" dependencies in the dev dependencies. "pts-framework[doc,test]", + "pydantic>=2.13,<3", "ruff", ] diff --git a/spikes/recipe_pydantic/__init__.py b/spikes/recipe_pydantic/__init__.py new file mode 100644 index 0000000..dfbf6e5 --- /dev/null +++ b/spikes/recipe_pydantic/__init__.py @@ -0,0 +1,28 @@ +"""Isolated Pydantic prototype for recipe-language version 2.""" + +from .models import Recipe +from .parser import ( + Diagnostic, + ParseResult, + RecipeParseError, + SourcePosition, + SourceSpan, + dump_recipe, + parse_recipe_file, + parse_recipe_text, +) +from .reference import render_json_schema, render_reference + +__all__ = [ + "Diagnostic", + "ParseResult", + "Recipe", + "RecipeParseError", + "SourcePosition", + "SourceSpan", + "dump_recipe", + "parse_recipe_file", + "parse_recipe_text", + "render_json_schema", + "render_reference", +] diff --git a/spikes/recipe_pydantic/models.py b/spikes/recipe_pydantic/models.py new file mode 100644 index 0000000..e616392 --- /dev/null +++ b/spikes/recipe_pydantic/models.py @@ -0,0 +1,380 @@ +"""Authoritative Pydantic model for the candidate recipe language. + +Field declarations intentionally own types, defaults, descriptions, examples, +serialization behavior, and JSON Schema. There is no parallel field registry. +""" + +from __future__ import annotations + +from typing import Annotated, Any, Literal, get_args + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic_core import PydanticCustomError + + +def described(description: str, *, example: Any = None, **kwargs: Any) -> Any: + """Small spelling helper; returned metadata still lives on each Field.""" + examples = None if example is None else [example] + return Field(description=description, examples=examples, **kwargs) + + +class RecipeModel(BaseModel): + """Strict, immutable base for all authorable structures.""" + + model_config = ConfigDict( + extra="forbid", + frozen=True, + strict=True, + populate_by_name=True, + validate_default=True, + ) + + +class DirectInput(RecipeModel): + """Provides a literal value.""" + + type: Literal["direct"] = described("Input source type.", example="direct") + value: Any = described("Literal input value.", example=1) + indexed: bool = described( + "Expand a list into indexed steps.", example=False, default=False, + exclude_if=lambda value: not value, + ) + + @model_validator(mode="after") + def indexed_values_are_lists(self) -> DirectInput: + if self.indexed and not isinstance(self.value, list): + raise PydanticCustomError( + "invalid_indexed_input", "Indexed direct input value must be a list." + ) + return self + + +class LocalInput(RecipeModel): + """Reads a sequence-local variable.""" + + type: Literal["local"] = described("Input source type.", example="local") + local_name: str = described("Local variable name.", example="local_value") + + +class GlobalInput(RecipeModel): + """Reads a recipe-global variable.""" + + type: Literal["global"] = described("Input source type.", example="global") + global_name: str = described("Global variable name.", example="global_value") + + +class MethodInput(RecipeModel): + """Resolves a method reference for the step.""" + + type: Literal["method"] = described("Input source type.", example="method") + value: Any = described("Method reference.", example="helper") + + +type InputMapping = Annotated[ + DirectInput | LocalInput | GlobalInput | MethodInput, + Field(discriminator="type"), +] + + +class PassFailOutput(RecipeModel): + """Interprets the output as a pass/fail verdict.""" + + type: Literal["passfail"] = described("Output mapping type.", example="passfail") + + +class EqualsOutput(RecipeModel): + """Passes when the output equals the configured value.""" + + type: Literal["equals"] = described("Output mapping type.", example="equals") + value: Any = described("Expected value.", example=3) + + +class RangeOutput(RecipeModel): + """Passes when the output is within an inclusive range.""" + + type: Literal["range"] = described("Output mapping type.", example="range") + minimum: Any = described("Minimum accepted value.", example=1, alias="min") + maximum: Any = described("Maximum accepted value.", example=4, alias="max") + + +class PassthroughOutput(RecipeModel): + """Uses the nested result without adding a verdict.""" + + type: Literal["passthrough"] = described("Output mapping type.", example="passthrough") + + +class LocalOutput(RecipeModel): + """Stores the output in a sequence-local variable.""" + + type: Literal["local"] = described("Output mapping type.", example="local") + local_name: str = described("Local destination variable.", example="saved") + + +class GlobalOutput(RecipeModel): + """Stores the output in a recipe-global variable.""" + + type: Literal["global"] = described("Output mapping type.", example="global") + global_name: str = described("Global destination variable.", example="saved") + + +class ImageOutput(RecipeModel): + """Publishes an image output for presentation.""" + + type: Literal["image"] = described("Output mapping type.", example="image") + + +type OutputMapping = Annotated[ + PassFailOutput + | EqualsOutput + | RangeOutput + | PassthroughOutput + | LocalOutput + | GlobalOutput + | ImageOutput, + Field(discriminator="type"), +] + + +class InternalSequenceReference(RecipeModel): + """Reference to another sequence in this recipe.""" + + type: Literal["internal"] = described("Reference kind.", example="internal") + name: str = described("Target sequence name.", example="Calibration") + + +class FileDestination(RecipeModel): + """Destination used by a file-loading step.""" + + type: Literal["local", "global"] = described("Variable scope.", example="local") + variable: str = described("Destination variable name.", example="selected_file") + + +class UploadFile(RecipeModel): + """One local-to-remote SSH upload pair.""" + + local: str = described("Local file or package resource.", example="bin/tool") + remote: str = described("Remote destination path.", example="/tmp/tool") + + +class CommonStep(RecipeModel): + """Fields shared by every authorable step.""" + + step_name: str = described("Human-readable step name.", example="Run test") + description: str = described("Purpose of the step.", example="Run a test operation.") + id: str | None = described("Optional stable step identifier.", example="test-1", default=None) + skip: bool = described("Skip execution.", example=False, default=False) + critical: bool = described( + "Stop on error when policy permits continuation.", example=False, default=False + ) + continue_on_error: bool = described("Per-step error policy.", example=False, default=False) + input_mapping: dict[str, InputMapping] = described( + "Named input sources.", example={}, default_factory=dict + ) + output_mapping: dict[str, OutputMapping] = described( + "Named verdicts and destinations.", example={}, default_factory=dict + ) + + +class PythonModuleStep(CommonStep): + """Calls a method or reads/writes an attribute in a Python module.""" + + steptype: Literal["PythonModuleStep"] = described( + "Canonical registered step type.", example="PythonModuleStep" + ) + action_type: Literal["method", "read_attribute", "write_attribute"] = described( + "Operation performed on the Python module.", example="method" + ) + module: str = described("Python module path.", example="tests.py") + method_name: str | None = described("Method name for method actions.", example="run", default=None) + + @model_validator(mode="after") + def method_actions_have_names(self) -> PythonModuleStep: + if self.action_type == "method" and not self.method_name: + raise PydanticCustomError( + "missing_method_name", "Method actions require method_name." + ) + return self + + +class SequenceStep(CommonStep): + """Runs another sequence as a step.""" + + steptype: Literal["SequenceStep"] = described( + "Canonical registered step type.", example="SequenceStep" + ) + sequence: InternalSequenceReference = described( + "Internal sequence reference.", example={"type": "internal", "name": "Calibration"} + ) + + +class UserInteractionStep(CommonStep): + """Displays an operator interaction prompt.""" + + steptype: Literal["UserInteractionStep"] = described( + "Canonical registered step type.", example="UserInteractionStep" + ) + + +class WaitStep(CommonStep): + """Waits for a non-negative duration in seconds.""" + + steptype: Literal["WaitStep"] = described("Canonical registered step type.", example="WaitStep") + + @model_validator(mode="after") + def has_wait_time(self) -> WaitStep: + if "wait_time" not in self.input_mapping: + raise PydanticCustomError("missing_required_input", "WaitStep requires input 'wait_time'.") + return self + + +class UserLoadingStep(CommonStep): + """Prompts the operator to select a file.""" + + steptype: Literal["UserLoadingStep"] = described( + "Canonical registered step type.", example="UserLoadingStep" + ) + file_save_location: FileDestination | None = described( + "Local or global destination for the selected file.", + example={"type": "local", "variable": "selected_file"}, + default=None, + ) + + +class UserRunMethodStep(CommonStep): + """Optionally runs a Python method after an operator response.""" + + steptype: Literal["UserRunMethodStep"] = described( + "Canonical registered step type.", example="UserRunMethodStep" + ) + trigger_response: str | list[Any] | dict[str, Any] | None = described( + "Operator response that triggers execution.", example="run", default=None + ) + action_type: str | None = described("Optional Python action type.", example="method", default=None) + module: str | None = described("Optional Python module path.", example="tests.py", default=None) + method_name: str | None = described("Optional Python method name.", example="run", default=None) + + +class UserWriteStep(CommonStep): + """Writes an operator-provided value to a configured destination.""" + + steptype: Literal["UserWriteStep"] = described( + "Canonical registered step type.", example="UserWriteStep" + ) + + +class SerialNumberStep(CommonStep): + """Captures the device serial number.""" + + steptype: Literal["SerialNumberStep"] = described( + "Canonical registered step type.", example="SerialNumberStep" + ) + + +class SSHConnectStep(CommonStep): + """Opens the SSH client stored in recipe globals.""" + + steptype: Literal["SSHConnectStep"] = described( + "Canonical registered step type.", example="SSHConnectStep" + ) + + +class SSHCloseStep(CommonStep): + """Closes the SSH client stored in recipe globals.""" + + steptype: Literal["SSHCloseStep"] = described( + "Canonical registered step type.", example="SSHCloseStep" + ) + + +class SSHUploadStep(CommonStep): + """Uploads files through an SSH connection.""" + + steptype: Literal["SSHUploadStep"] = described( + "Canonical registered step type.", example="SSHUploadStep" + ) + files: list[UploadFile] = described( + "Local and remote file pairs to upload.", + example=[{"local": "bin/tool", "remote": "/tmp/tool"}], + ) + permissions: int | str | None = described( + "Optional remote permissions.", example="0755", default=None + ) + skip_if_sha256_match: bool = described( + "Skip files whose remote checksum matches.", example=False, default=False + ) + local_package: str | None = described( + "Optional package containing local resources.", example="my_package", default=None + ) + + +type Step = Annotated[ + PythonModuleStep + | SequenceStep + | UserInteractionStep + | WaitStep + | UserLoadingStep + | UserRunMethodStep + | UserWriteStep + | SerialNumberStep + | SSHConnectStep + | SSHCloseStep + | SSHUploadStep, + Field(discriminator="steptype"), +] + + +class RecipeHeader(RecipeModel): + """The first YAML document, identifying a recipe and its entry sequence.""" + + name: str = described("Human-readable recipe name.", example="Hardware acceptance") + version: str = described("Version of this recipe.", example="1.0") + recipe_version: Literal["2.0.0"] = described( + "Version of the recipe language contract.", example="2.0.0" + ) + description: str = described("Purpose of the recipe.", example="Acceptance tests.") + main_sequence: str = described("Sequence where execution begins.", example="Main") + globals: dict[str, Any] = described("Recipe-wide variables.", example={}) + continue_on_error: bool | None = described( + "Recipe-wide error policy.", example=False, default=None + ) + report: Literal["overwrite", "append"] = described( + "Report file mode.", example="overwrite", default="overwrite" + ) + report_name_include_serial: bool = described( + "Include the serial number in the report name.", example=False, default=False + ) + test_package: str | None = described( + "Package containing recipe test modules.", example="acceptance", default=None + ) + + +class Sequence(RecipeModel): + """One named executable sequence document.""" + + sequence_name: str = described("Unique sequence name.", example="Main") + description: str = described("Purpose of the sequence.", example="Main sequence.") + parameters: dict[str, Any] = described("Reserved sequence input metadata.", example={}) + outputs: dict[str, Any] = described("Reserved sequence output metadata.", example={}) + locals: dict[str, Any] = described("Variables local to the sequence.", example={}) + setup_steps: list[Step] = described("Steps run before the main steps.", example=[]) + steps: list[Step] = described("Ordered main steps.", example=[]) + teardown_steps: list[Step] = described("Steps run during teardown.", example=[]) + + +class Recipe(RecipeModel): + """Aggregate typed recipe used by tooling and JSON Schema consumers.""" + + header: RecipeHeader = described("Recipe header document.") + sequences: list[Sequence] = described("Sequence documents.", min_length=1) + + +def _union_models(annotation: Any) -> tuple[type[RecipeModel], ...]: + """Expose union members for generators without a second registry.""" + annotation = getattr(annotation, "__value__", annotation) + annotated_union = get_args(annotation)[0] + return get_args(annotated_union) + + +STEP_MODELS = _union_models(Step) +INPUT_MODELS = _union_models(InputMapping) +OUTPUT_MODELS = _union_models(OutputMapping) diff --git a/spikes/recipe_pydantic/parser.py b/spikes/recipe_pydantic/parser.py new file mode 100644 index 0000000..83bf4cc --- /dev/null +++ b/spikes/recipe_pydantic/parser.py @@ -0,0 +1,534 @@ +"""Safe YAML adapter and semantic validation for the Pydantic v2 spike.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml +from pydantic import ValidationError + +from .models import ( + DirectInput, + EqualsOutput, + PassFailOutput, + PassthroughOutput, + RangeOutput, + Recipe, + RecipeHeader, + Sequence, + SequenceStep, +) + +type RecipePath = tuple[str | int, ...] + + +@dataclass(frozen=True) +class SourcePosition: + """A one-based source position with a zero-based character offset.""" + + line: int + column: int + offset: int + + +@dataclass(frozen=True) +class SourceSpan: + """Half-open source range.""" + + start: SourcePosition + end: SourcePosition + + +@dataclass(frozen=True) +class Diagnostic: + """Source-aware recipe finding, compatible with the PyPTS envelope.""" + + code: str + message: str + path: RecipePath = () + severity: str = "error" + source_name: str | None = None + span: SourceSpan | None = None + + +class RecipeParseError(ValueError): + """Raised when a caller requires a recipe from an unsuccessful parse.""" + + def __init__(self, diagnostics: tuple[Diagnostic, ...]): + self.diagnostics = diagnostics + errors = sum(item.severity == "error" for item in diagnostics) + super().__init__(f"Recipe parsing failed with {errors} error(s).") + + +@dataclass(frozen=True) +class ParseResult: + recipe: Recipe | None + diagnostics: tuple[Diagnostic, ...] = () + + @property + def errors(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "error") + + @property + def warnings(self) -> tuple[Diagnostic, ...]: + return tuple(item for item in self.diagnostics if item.severity == "warning") + + @property + def is_valid(self) -> bool: + return self.recipe is not None and not self.errors + + def require_recipe(self) -> Recipe: + if not self.is_valid: + raise RecipeParseError(self.diagnostics) + assert self.recipe is not None + return self.recipe + + +def _position(mark: yaml.error.Mark) -> SourcePosition: + return SourcePosition(mark.line + 1, mark.column + 1, mark.index) + + +def _span(node: yaml.Node) -> SourceSpan: + return SourceSpan(_position(node.start_mark), _position(node.end_mark)) + + +def _mark_span(mark: yaml.error.Mark | None) -> SourceSpan | None: + if mark is None: + return None + position = _position(mark) + return SourceSpan(position, position) + + +def _nearest_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: + candidate = path + while candidate: + if candidate in spans: + return spans[candidate] + candidate = candidate[:-1] + return spans.get(()) + + +def _diagnostic( + code: str, + message: str, + path: RecipePath, + source_name: str, + spans: Mapping[RecipePath, SourceSpan], +) -> Diagnostic: + return Diagnostic(code, message, path, source_name=source_name, span=_nearest_span(path, spans)) + + +def _index_nodes( + node: yaml.Node, + path: RecipePath, + spans: dict[RecipePath, SourceSpan], + diagnostics: list[Diagnostic], + source_name: str, + active: set[int], +) -> None: + spans[path] = _span(node) + identity = id(node) + if identity in active: + diagnostics.append(Diagnostic( + "recursive-alias", + "Recursive YAML aliases are not supported.", + path, + source_name=source_name, + span=_span(node), + )) + return + active.add(identity) + try: + if isinstance(node, yaml.MappingNode): + seen: set[tuple[str, str]] = set() + for key_node, value_node in node.value: + key_identity = (key_node.tag, repr(key_node.value)) + key: str | int = key_node.value if isinstance(key_node, yaml.ScalarNode) else repr(key_node.value) + child = path + (key,) + if key_identity in seen: + diagnostics.append(Diagnostic( + "duplicate-key", + f"Duplicate YAML key '{key}'.", + child, + source_name=source_name, + span=_span(key_node), + )) + seen.add(key_identity) + _index_nodes(value_node, child, spans, diagnostics, source_name, active) + elif isinstance(node, yaml.SequenceNode): + for index, child_node in enumerate(node.value): + _index_nodes( + child_node, path + (index,), spans, diagnostics, source_name, active + ) + finally: + active.remove(identity) + + +_MODEL_TAGS = { + "direct", "local", "global", "method", "passfail", "equals", "range", + "passthrough", "image", "PythonModuleStep", "SequenceStep", + "UserInteractionStep", "WaitStep", "UserLoadingStep", "UserRunMethodStep", + "UserWriteStep", "SerialNumberStep", "SSHConnectStep", "SSHCloseStep", + "SSHUploadStep", +} +_CANONICAL_STEPS = {tag for tag in _MODEL_TAGS if tag.endswith("Step")} + + +def _clean_location(location: tuple[Any, ...]) -> RecipePath: + cleaned: list[str | int] = [] + for index, item in enumerate(location): + is_step_tag = ( + item in _CANONICAL_STEPS + and index >= 2 + and location[index - 2] in {"setup_steps", "steps", "teardown_steps"} + and isinstance(location[index - 1], int) + ) + is_mapping_tag = ( + item in _MODEL_TAGS + and index >= 2 + and location[index - 2] in {"input_mapping", "output_mapping"} + ) + if not is_step_tag and not is_mapping_tag: + cleaned.append(item) + return tuple(cleaned) + + +def _pydantic_diagnostic( + error: dict[str, Any], + prefix: RecipePath, + source_name: str, + spans: Mapping[RecipePath, SourceSpan], +) -> Diagnostic: + location = prefix + _clean_location(tuple(error.get("loc", ()))) + kind = error["type"] + context = error.get("ctx") or {} + input_value = error.get("input") + code = "invalid-field" + message = error["msg"] + + if kind == "missing": + code = "missing-field" + elif kind == "extra_forbidden": + field = location[-1] if location else "field" + if field == "serial_number": + code = "removed-sequence-field" + message = "Sequence field 'serial_number' was removed in recipe language 2.0.0." + else: + code = "unknown-field" + message = f"Unknown field '{field}'." + elif kind == "union_tag_not_found": + discriminator = str(context.get("discriminator", "")) + if "steptype" in discriminator: + code = "missing-step-type" + location += ("steptype",) + message = "Step requires canonical 'steptype'." + elif "output_mapping" in location: + code = "missing-output-type" + location += ("type",) + message = "Output mapping requires an explicit 'type'." + else: + code = "missing-input-type" + location += ("type",) + message = "Mapping requires an explicit 'type' in recipe language 2.0.0." + elif kind == "union_tag_invalid": + discriminator = str(context.get("discriminator", "")) + tag = context.get("tag") + if "steptype" in discriminator: + location += ("steptype",) + canonical = next( + (candidate for candidate in _CANONICAL_STEPS if candidate.casefold() == str(tag).casefold()), + None, + ) + if canonical: + code = "noncanonical-step-type" + message = f"Use canonical step type '{canonical}' instead of '{tag}'." + else: + code = "unknown-step-type" + message = f"Unknown step type '{tag}'." + else: + location += ("type",) + if "output_mapping" in location: + code = "unknown-output-type" + message = f"Unknown output mapping type '{tag}'." + else: + code = "unknown-input-type" + message = f"Unknown input mapping type '{tag}'." + elif kind == "literal_error" and location[-1:] == ("recipe_version",): + code = "unsupported-recipe-version" + message = f"Recipe language version {input_value!r} is unsupported; expected '2.0.0'." + elif kind == "literal_error": + code = "invalid-field-value" + elif kind in {"bool_type", "string_type", "int_type", "list_type", "dict_type", "model_type"}: + code = "invalid-field-type" + elif kind == "invalid_indexed_input": + code = "invalid-indexed-input" + elif kind == "missing_method_name": + code = "missing-method-name" + location += ("method_name",) + elif kind == "missing_required_input": + code = "missing-required-input" + location += ("input_mapping", "wait_time") + elif kind == "too_short" and prefix == () and location[-1:] == ("sequences",): + code = "missing-sequence" + + return _diagnostic(code, message, location, source_name, spans) + + +def _validation_diagnostics( + error: ValidationError, + prefix: RecipePath, + source_name: str, + spans: Mapping[RecipePath, SourceSpan], +) -> list[Diagnostic]: + return [ + _pydantic_diagnostic(item, prefix, source_name, spans) + for item in error.errors(include_url=False) + ] + + +def _all_steps(sequence: Sequence): + for section_name in ("setup_steps", "steps", "teardown_steps"): + for index, step in enumerate(getattr(sequence, section_name)): + yield section_name, index, step + + +def _semantic_diagnostics( + header: RecipeHeader | None, + sequences: list[tuple[int, Sequence]], + source_name: str, + spans: Mapping[RecipePath, SourceSpan], + *, + complete_sequences: bool = True, +) -> list[Diagnostic]: + """Rules that cannot be expressed by one structural Pydantic model.""" + diagnostics: list[Diagnostic] = [] + by_name: dict[str, tuple[int, Sequence]] = {} + for document_index, sequence in sequences: + path = (document_index, "sequence_name") + if sequence.sequence_name in by_name: + diagnostics.append(_diagnostic( + "duplicate-sequence", + f"Duplicate sequence '{sequence.sequence_name}'.", + path, + source_name, + spans, + )) + else: + by_name[sequence.sequence_name] = (document_index, sequence) + + if complete_sequences and header is not None and header.main_sequence not in by_name: + diagnostics.append(_diagnostic( + "unknown-main-sequence", + f"Main sequence '{header.main_sequence}' does not exist.", + (0, "main_sequence"), + source_name, + spans, + )) + + verdict_types = (PassFailOutput, EqualsOutput, RangeOutput, PassthroughOutput) + for document_index, sequence in sequences: + flattened = list(_all_steps(sequence)) + for section, index, step in flattened: + step_path = (document_index, section, index) + if isinstance(step, SequenceStep) and step.sequence.name not in by_name: + diagnostics.append(_diagnostic( + "unknown-sequence-reference", + f"Sequence '{sequence.sequence_name}' references unknown sequence " + f"'{step.sequence.name}'.", + step_path + ("sequence", "name"), + source_name, + spans, + )) + + indexed_lengths = [ + len(value.value) + for value in step.input_mapping.values() + if isinstance(value, DirectInput) and value.indexed + ] + if len(set(indexed_lengths)) > 1: + diagnostics.append(_diagnostic( + "unequal-indexed-inputs", + "Indexed input lists must have equal lengths.", + step_path + ("input_mapping",), + source_name, + spans, + )) + + verdicts = [ + value for value in step.output_mapping.values() if isinstance(value, verdict_types) + ] + if any(isinstance(value, PassthroughOutput) for value in verdicts) and len(verdicts) != 1: + diagnostics.append(_diagnostic( + "mixed-passthrough", + "'passthrough' must be the sole verdict mapping.", + step_path + ("output_mapping",), + source_name, + spans, + )) + + ssh_steps = [item for item in flattened if item[2].steptype.startswith("SSH")] + if ssh_steps and header is not None: + for required in ("ssh_client", "host", "user", "port"): + if required not in header.globals: + diagnostics.append(_diagnostic( + "missing-ssh-global", + f"SSH step requires global '{required}'.", + (0, "globals", required), + source_name, + spans, + )) + if "password" not in header.globals and "private_key" not in header.globals: + diagnostics.append(_diagnostic( + "missing-ssh-credential", + "SSH steps require password or private_key global.", + (0, "globals"), + source_name, + spans, + )) + + connected = False + unclosed_connect: tuple[str, int] | None = None + for section, index, step in flattened: + if step.steptype == "SSHConnectStep": + connected = True + unclosed_connect = (section, index) + elif step.steptype == "SSHUploadStep" and not connected: + diagnostics.append(_diagnostic( + "missing-ssh-connect", + f"Sequence '{sequence.sequence_name}' uploads before an SSH connection.", + (document_index, section, index), + source_name, + spans, + )) + elif step.steptype == "SSHCloseStep": + connected = False + unclosed_connect = None + if unclosed_connect is not None: + diagnostics.append(_diagnostic( + "missing-ssh-close", + f"Sequence '{sequence.sequence_name}' opens SSH without a later close.", + (document_index, "teardown_steps"), + source_name, + spans, + )) + return diagnostics + + +def parse_recipe_text(text: str, source_name: str = "") -> ParseResult: + """Parse candidate recipe-language 2 YAML without runtime or GUI imports.""" + if not isinstance(text, str): + return ParseResult(None, (Diagnostic( + "invalid-source", "Recipe source must be text.", source_name=source_name + ),)) + if not text.strip(): + return ParseResult(None, (Diagnostic( + "empty-recipe", "A recipe requires a header and at least one sequence.", + source_name=source_name, + ),)) + + try: + nodes = list(yaml.compose_all(text, Loader=yaml.SafeLoader)) + except yaml.YAMLError as error: + mark = getattr(error, "problem_mark", None) + return ParseResult(None, (Diagnostic( + "yaml-syntax-error", str(error), source_name=source_name, span=_mark_span(mark) + ),)) + + spans: dict[RecipePath, SourceSpan] = {} + diagnostics: list[Diagnostic] = [] + for index, node in enumerate(nodes): + if node is not None: + _index_nodes(node, (index,), spans, diagnostics, source_name, set()) + + try: + documents = list(yaml.safe_load_all(text)) + except yaml.YAMLError as error: + mark = getattr(error, "problem_mark", None) + code = "unsafe-yaml" if isinstance(error, yaml.constructor.ConstructorError) else "yaml-construction-error" + diagnostics.append(Diagnostic(code, str(error), source_name=source_name, span=_mark_span(mark))) + return ParseResult(None, tuple(diagnostics)) + + if not documents or all(document is None for document in documents): + diagnostics.append(Diagnostic( + "empty-recipe", "A recipe requires a header and at least one sequence.", + source_name=source_name, + )) + return ParseResult(None, tuple(diagnostics)) + + header: RecipeHeader | None = None + semantic_header: RecipeHeader | None = None + raw_header = documents[0] + try: + header = RecipeHeader.model_validate(raw_header) + semantic_header = header + except ValidationError as error: + diagnostics.extend(_validation_diagnostics(error, (0,), source_name, spans)) + if isinstance(raw_header, dict) and raw_header.get("recipe_version") != "2.0.0": + candidate = dict(raw_header) + candidate["recipe_version"] = "2.0.0" + try: + semantic_header = RecipeHeader.model_validate(candidate) + except ValidationError: + pass + + sequences: list[tuple[int, Sequence]] = [] + complete_sequences = True + for index, document in enumerate(documents[1:], start=1): + try: + sequences.append((index, Sequence.model_validate(document))) + except ValidationError as error: + complete_sequences = False + diagnostics.extend(_validation_diagnostics(error, (index,), source_name, spans)) + + if len(documents) == 1: + diagnostics.append(_diagnostic( + "missing-sequence", "A recipe requires at least one sequence.", (0,), source_name, spans + )) + + diagnostics.extend(_semantic_diagnostics( + semantic_header, + sequences, + source_name, + spans, + complete_sequences=complete_sequences, + )) + if diagnostics: + return ParseResult(None, tuple(diagnostics)) + assert header is not None + recipe = Recipe(header=header, sequences=[sequence for _, sequence in sequences]) + return ParseResult(recipe) + + +def parse_recipe_file(path: str | Path, encoding: str = "utf-8") -> ParseResult: + """Read and parse a candidate recipe file.""" + source_path = Path(path) + try: + text = source_path.read_text(encoding=encoding) + except (OSError, UnicodeError) as error: + return ParseResult(None, (Diagnostic( + "file-read-error", f"Could not read recipe: {error}", source_name=str(source_path) + ),)) + return parse_recipe_text(text, str(source_path)) + + +def dump_recipe(recipe: Recipe) -> str: + """Serialize a typed recipe as canonical multi-document YAML.""" + if not isinstance(recipe, Recipe): + raise TypeError("dump_recipe expects a Recipe") + documents = [ + recipe.header.model_dump(mode="python", by_alias=True, exclude_none=True), + *[ + sequence.model_dump(mode="python", by_alias=True, exclude_none=True) + for sequence in recipe.sequences + ], + ] + return yaml.safe_dump_all( + documents, + explicit_start=True, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + ) diff --git a/spikes/recipe_pydantic/reference.py b/spikes/recipe_pydantic/reference.py new file mode 100644 index 0000000..a66bbb0 --- /dev/null +++ b/spikes/recipe_pydantic/reference.py @@ -0,0 +1,128 @@ +"""JSON Schema and compact RST renderers driven only by Pydantic metadata.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from pydantic import BaseModel +from pydantic_core import PydanticUndefined + +from .models import ( + INPUT_MODELS, + OUTPUT_MODELS, + STEP_MODELS, + FileDestination, + InternalSequenceReference, + Recipe, + RecipeHeader, + Sequence, + UploadFile, +) + +ROOT = Path(__file__).parent +SCHEMA_PATH = ROOT / "recipe.schema.json" +REFERENCE_PATH = ROOT / "recipe_reference.rst" + + +def render_json_schema() -> str: + """Render deterministic JSON Schema for the aggregate typed model.""" + schema = Recipe.model_json_schema(by_alias=True, mode="validation") + schema["$comment"] = ( + "SPDX-FileCopyrightText: 2026 CERN ; " + "SPDX-License-Identifier: LGPL-2.1-or-later" + ) + return json.dumps( + schema, + indent=2, + sort_keys=True, + ) + "\n" + + +def _annotation_name(annotation: Any) -> str: + text = str(annotation).replace("typing.", "") + text = text.replace("", "") + text = re.sub(r"(?:[\w.]*recipe_pydantic\.models|__main__)\.", "", text) + return text.replace("NoneType", "None") + + +def _example(field: Any) -> Any: + if field.examples: + return field.examples[0] + if field.default is not PydanticUndefined: + return field.default + return PydanticUndefined + + +def _model_block(model: type[BaseModel], anchor: str) -> list[str]: + title = model.__name__ + lines = [f".. _{anchor}:", "", title, "~" * len(title), ""] + description = (model.__doc__ or "").strip() + if description: + lines.extend([description, ""]) + lines.extend([".. list-table:: Fields", " :header-rows: 1", "", " * - Field", " - Type", " - Required/default", " - Description / example"]) + for name, field in model.model_fields.items(): + public_name = field.alias or name + default = "required" if field.is_required() else f"default ``{field.default!r}``" + example = _example(field) + detail = field.description or "" + if example is not PydanticUndefined: + detail += f" Example: ``{example!r}``." + lines.extend([ + f" * - ``{public_name}``", + f" - ``{_annotation_name(field.annotation)}``", + f" - {default}", + f" - {detail}", + ]) + lines.append("") + return lines + + +def render_reference() -> str: + """Render the review-oriented recipe reference from model metadata.""" + lines = [ + ".. SPDX-FileCopyrightText: 2026 CERN ", + ".. SPDX-License-Identifier: LGPL-2.1-or-later", + ".. This file is generated by the Pydantic recipe spike.", + "", + "Pydantic Recipe Language 2.0.0", + "================================", + "", + "This candidate reference is generated directly from strict, frozen Pydantic models.", + "", + "Documents", + "---------", + "", + ] + lines.extend(_model_block(RecipeHeader, "recipe-header")) + lines.extend(_model_block(Sequence, "recipe-sequence")) + + lines.extend(["Nested structures", "-----------------", ""]) + for model in (InternalSequenceReference, FileDestination, UploadFile): + lines.extend(_model_block(model, f"recipe-structure-{model.__name__.lower()}")) + + lines.extend(["Authorable steps", "----------------", ""]) + for model in STEP_MODELS: + lines.extend(_model_block(model, f"recipe-step-{model.__name__.lower()}")) + + lines.extend(["Input mappings", "--------------", ""]) + for model in INPUT_MODELS: + kind = model.model_fields["type"].examples[0] + lines.extend(_model_block(model, f"recipe-input-{kind}")) + + lines.extend(["Output mappings", "---------------", ""]) + for model in OUTPUT_MODELS: + kind = model.model_fields["type"].examples[0] + lines.extend(_model_block(model, f"recipe-output-{kind}")) + return "\n".join(lines).rstrip() + "\n" + + +def write_artifacts() -> None: + SCHEMA_PATH.write_text(render_json_schema(), encoding="utf-8") + REFERENCE_PATH.write_text(render_reference(), encoding="utf-8") + + +if __name__ == "__main__": + write_artifacts() diff --git a/spikes/recipe_pydantic/test_recipe_pydantic.py b/spikes/recipe_pydantic/test_recipe_pydantic.py new file mode 100644 index 0000000..a0d62d3 --- /dev/null +++ b/spikes/recipe_pydantic/test_recipe_pydantic.py @@ -0,0 +1,404 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Executable evaluation suite for the isolated Pydantic recipe spike.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +from pypts.recipe_parser import dump_recipe as dump_v1 +from pypts.recipe_parser import parse_recipe_file as parse_v1 + +from .models import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS +from .parser import RecipeParseError, dump_recipe, parse_recipe_file, parse_recipe_text +from .reference import REFERENCE_PATH, SCHEMA_PATH, render_json_schema, render_reference + +ROOT = Path(__file__).parents[2] +RECIPES = ROOT / "src" / "pypts" / "recipes" + + +def header(**updates): + value = { + "name": "Pydantic spike", + "version": "1.0", + "recipe_version": "2.0.0", + "description": "Candidate recipe.", + "main_sequence": "Main", + "globals": {}, + } + value.update(updates) + return value + + +def sequence(steps=None, **updates): + value = { + "sequence_name": "Main", + "description": "Main sequence.", + "parameters": {}, + "outputs": {}, + "locals": {}, + "setup_steps": [], + "steps": steps or [], + "teardown_steps": [], + } + value.update(updates) + return value + + +def source(*documents): + return yaml.safe_dump_all(documents, explicit_start=True, sort_keys=False) + + +def common(kind, **updates): + value = { + "steptype": kind, + "step_name": kind, + "description": f"Exercise {kind}.", + "input_mapping": {}, + "output_mapping": {}, + } + value.update(updates) + return value + + +STEP_EXAMPLES = { + "PythonModuleStep": common( + "PythonModuleStep", action_type="method", module="tests.py", method_name="run" + ), + "SequenceStep": common( + "SequenceStep", sequence={"type": "internal", "name": "Target"} + ), + "UserInteractionStep": common("UserInteractionStep"), + "WaitStep": common( + "WaitStep", input_mapping={"wait_time": {"type": "direct", "value": 0}} + ), + "UserLoadingStep": common( + "UserLoadingStep", + file_save_location={"type": "local", "variable": "selected"}, + ), + "UserRunMethodStep": common( + "UserRunMethodStep", + trigger_response="run", + action_type="method", + module="tests.py", + method_name="run", + ), + "UserWriteStep": common("UserWriteStep"), + "SerialNumberStep": common("SerialNumberStep"), + "SSHConnectStep": common("SSHConnectStep"), + "SSHCloseStep": common("SSHCloseStep"), + "SSHUploadStep": common( + "SSHUploadStep", + files=[{"local": "bin/tool", "remote": "/tmp/tool"}], + permissions="0755", + skip_if_sha256_match=True, + local_package="fixtures", + ), +} + + +def recipe_for_step(step): + globals_value = { + "ssh_client": None, + "host": "target", + "user": "root", + "port": 22, + "password": "secret", + } + main = sequence([step]) + documents = [header(globals=globals_value), main] + if step["steptype"] == "SequenceStep": + documents.append(sequence(sequence_name="Target")) + elif step["steptype"] == "SSHUploadStep": + main["setup_steps"] = [common("SSHConnectStep")] + main["teardown_steps"] = [common("SSHCloseStep")] + elif step["steptype"] == "SSHConnectStep": + main["teardown_steps"] = [common("SSHCloseStep")] + return source(*documents) + + +@pytest.mark.parametrize("model", STEP_MODELS, ids=lambda model: model.__name__) +def test_every_step_validates_serializes_and_reparses(model): + first = parse_recipe_text(recipe_for_step(STEP_EXAMPLES[model.__name__])) + assert first.is_valid, first.errors + second = parse_recipe_text(dump_recipe(first.require_recipe())) + assert second.is_valid, second.errors + assert second.recipe == first.recipe + + +INPUT_EXAMPLES = { + "DirectInput": {"type": "direct", "value": [1, 2], "indexed": True}, + "LocalInput": {"type": "local", "local_name": "local_value"}, + "GlobalInput": {"type": "global", "global_name": "global_value"}, + "MethodInput": {"type": "method", "value": "helper"}, +} + + +@pytest.mark.parametrize("model", INPUT_MODELS, ids=lambda model: model.__name__) +def test_every_input_mapping_validates_serializes_and_reparses(model): + step = STEP_EXAMPLES["PythonModuleStep"] | { + "input_mapping": {"example": INPUT_EXAMPLES[model.__name__]} + } + first = parse_recipe_text(recipe_for_step(step)) + assert isinstance(first.require_recipe().sequences[0].steps[0].input_mapping["example"], model) + assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + + +OUTPUT_EXAMPLES = { + "PassFailOutput": {"type": "passfail"}, + "EqualsOutput": {"type": "equals", "value": 3}, + "RangeOutput": {"type": "range", "min": 1, "max": 4}, + "PassthroughOutput": {"type": "passthrough"}, + "LocalOutput": {"type": "local", "local_name": "saved"}, + "GlobalOutput": {"type": "global", "global_name": "saved"}, + "ImageOutput": {"type": "image"}, +} + + +@pytest.mark.parametrize("model", OUTPUT_MODELS, ids=lambda model: model.__name__) +def test_every_output_mapping_validates_serializes_and_reparses(model): + step = STEP_EXAMPLES["PythonModuleStep"] | { + "output_mapping": {"example": OUTPUT_EXAMPLES[model.__name__]} + } + first = parse_recipe_text(recipe_for_step(step)) + assert isinstance(first.require_recipe().sequences[0].steps[0].output_mapping["example"], model) + assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + + +def test_defaults_are_typed_dumped_and_models_are_frozen(): + recipe = parse_recipe_text(recipe_for_step(STEP_EXAMPLES["UserInteractionStep"])).require_recipe() + step = recipe.sequences[0].steps[0] + assert step.skip is step.critical is step.continue_on_error is False + assert recipe.header.report == "overwrite" + dumped = dump_recipe(recipe) + assert "report: overwrite" in dumped and "skip: false" in dumped + with pytest.raises(ValidationError): + step.skip = True + + +def test_strict_types_unknown_fields_and_structural_rules_are_rejected(): + bad = STEP_EXAMPLES["PythonModuleStep"] | { + "skip": 0, + "surprise": True, + "method_name": None, + } + codes = {item.code for item in parse_recipe_text(recipe_for_step(bad)).errors} + assert {"invalid-field-type", "unknown-field"} <= codes + + missing_method = STEP_EXAMPLES["PythonModuleStep"] | {"method_name": None} + assert "missing-method-name" in { + item.code for item in parse_recipe_text(recipe_for_step(missing_method)).errors + } + + wait = common("WaitStep") + assert "missing-required-input" in { + item.code for item in parse_recipe_text(recipe_for_step(wait)).errors + } + + nested = STEP_EXAMPLES["PythonModuleStep"] | { + "input_mapping": {"local": {"type": "local", "local_name": 1}} + } + finding = next( + item for item in parse_recipe_text(recipe_for_step(nested)).errors + if item.code == "invalid-field-type" + ) + assert finding.path[-2:] == ("local", "local_name") + + +def test_discriminators_are_explicit_and_canonical(): + lowercase = common( + "waitstep", input_mapping={"wait_time": {"type": "direct", "value": 1}} + ) + omitted_type = STEP_EXAMPLES["WaitStep"] | { + "input_mapping": {"wait_time": {"value": 1}} + } + unknown = common("InventedStep") + missing_output_type = STEP_EXAMPLES["PythonModuleStep"] | { + "output_mapping": {"result": {"value": 1}} + } + assert {item.code for item in parse_recipe_text(recipe_for_step(lowercase)).errors} == { + "noncanonical-step-type" + } + assert {item.code for item in parse_recipe_text(recipe_for_step(omitted_type)).errors} == { + "missing-input-type" + } + assert {item.code for item in parse_recipe_text(recipe_for_step(unknown)).errors} == { + "unknown-step-type" + } + assert { + item.code for item in parse_recipe_text(recipe_for_step(missing_output_type)).errors + } == {"missing-output-type"} + + +def test_v1_migration_errors_are_aggregated_across_documents(): + legacy = header(recipe_version="1.0.0") + first = sequence( + [common("waitstep", input_mapping={"wait_time": {"value": 1}})], + serial_number=12, + ) + second = sequence( + [STEP_EXAMPLES["WaitStep"] | { + "input_mapping": {"wait_time": {"value": 1}} + }], + sequence_name="Other", + serial_number="old", + ) + result = parse_recipe_text(source(legacy, first, second), "legacy.yml") + codes = [item.code for item in result.errors] + assert codes.count("removed-sequence-field") == 2 + assert {"unsupported-recipe-version", "noncanonical-step-type", "missing-input-type"} <= set(codes) + assert all(item.source_name == "legacy.yml" and item.span is not None for item in result.errors) + + +def test_source_spans_point_to_fields_and_nearest_parent(): + text = source(header(main_sequence="Missing"), sequence()) + result = parse_recipe_text(text, "broken.yml") + finding = next(item for item in result.errors if item.code == "unknown-main-sequence") + expected = next( + index for index, line in enumerate(text.splitlines(), start=1) + if line.startswith("main_sequence:") + ) + assert finding.source_name == "broken.yml" + assert finding.span is not None and finding.span.start.line == expected + + missing = source(header(), sequence()).replace("description: Main sequence.\n", "") + finding = next( + item for item in parse_recipe_text(missing).errors + if item.code == "missing-field" and item.path[-1] == "description" + ) + assert finding.span is not None and finding.span.start.line > 1 + + +def test_yaml_failures_duplicate_keys_and_recursive_aliases(): + malformed = parse_recipe_text("name: [unterminated") + unsafe = parse_recipe_text("!!python/object:builtins.object {}") + duplicate = parse_recipe_text( + source(header(), sequence()).replace("name: Pydantic spike", "name: First\nname: Second") + ) + recursive = parse_recipe_text("---\n&a {name: *a}\n") + assert {item.code for item in malformed.errors} == {"yaml-syntax-error"} + assert "unsafe-yaml" in {item.code for item in unsafe.errors} + assert "duplicate-key" in {item.code for item in duplicate.errors} + assert "recursive-alias" in {item.code for item in recursive.errors} + + +def test_file_api_empty_sources_and_require_recipe(tmp_path): + path = tmp_path / "recipe.yml" + path.write_text(source(header(), sequence()), encoding="utf-8") + assert parse_recipe_file(path).is_valid + assert parse_recipe_file(tmp_path / "missing.yml").errors[0].code == "file-read-error" + assert parse_recipe_text(None).errors[0].code == "invalid-source" + assert parse_recipe_text(" # comment only\n").errors[0].code == "empty-recipe" + result = parse_recipe_text("") + with pytest.raises(RecipeParseError) as error: + result.require_recipe() + assert error.value.diagnostics == result.diagnostics + + +def test_cross_document_semantic_rules_report_custom_codes(): + nested = common( + "SequenceStep", + sequence={"type": "internal", "name": "Missing"}, + input_mapping={ + "left": {"type": "direct", "value": [1], "indexed": True}, + "right": {"type": "direct", "value": [1, 2], "indexed": True}, + }, + output_mapping={ + "result": {"type": "passthrough"}, + "passed": {"type": "passfail"}, + }, + ) + duplicate = sequence(sequence_name="Main") + result = parse_recipe_text(source(header(), sequence([nested]), duplicate)) + assert { + "duplicate-sequence", + "unknown-sequence-reference", + "unequal-indexed-inputs", + "mixed-passthrough", + } <= {item.code for item in result.errors} + + +def test_ssh_context_and_ordering_are_semantic_rules(): + upload = STEP_EXAMPLES["SSHUploadStep"] + unclosed = sequence([upload], setup_steps=[common("SSHConnectStep")]) + result = parse_recipe_text(source(header(), unclosed)) + codes = {item.code for item in result.errors} + assert {"missing-ssh-global", "missing-ssh-credential", "missing-ssh-close"} <= codes + + before_connect = sequence([upload, common("SSHConnectStep")], teardown_steps=[common("SSHCloseStep")]) + assert "missing-ssh-connect" in { + item.code for item in parse_recipe_text(source(header(), before_connect)).errors + } + + +@pytest.mark.parametrize( + "path", + sorted( + path for path in RECIPES.glob("*.yml") + if path.name != "subsequence_executions_draft.yml" + ), +) +def test_normalized_bundled_corpus_passes_as_v2(path): + legacy = parse_v1(path) + assert legacy.is_valid, legacy.errors + normalized = dump_v1(legacy.require_recipe()).replace( + "recipe_version: 1.0.0", "recipe_version: 2.0.0" + ) + result = parse_recipe_text(normalized, f"normalized:{path.name}") + assert result.is_valid, result.errors + assert parse_recipe_text(dump_recipe(result.require_recipe())).recipe == result.recipe + + +def test_raw_legacy_corpus_exposes_migration_diagnostics(): + results = [ + parse_recipe_file(path) + for path in RECIPES.glob("*.yml") + if path.name != "subsequence_executions_draft.yml" + ] + assert all("unsupported-recipe-version" in {item.code for item in result.errors} for result in results) + all_codes = {item.code for result in results for item in result.errors} + assert {"noncanonical-step-type", "missing-input-type", "removed-sequence-field"} <= all_codes + + +def test_generated_schema_and_reference_are_complete_and_current(): + schema_text = render_json_schema() + reference = render_reference() + assert schema_text == SCHEMA_PATH.read_text(encoding="utf-8") + assert reference == REFERENCE_PATH.read_text(encoding="utf-8") + definitions = json.loads(schema_text)["$defs"] + for model in STEP_MODELS + INPUT_MODELS + OUTPUT_MODELS: + assert model.__name__ in definitions + kind = model.model_fields.get("steptype") or model.model_fields["type"] + anchor_kind = kind.examples[0].lower() + group = "step" if model in STEP_MODELS else "input" if model in INPUT_MODELS else "output" + assert reference.count(f".. _recipe-{group}-{anchor_kind}:") == 1 + + report = STEP_MODELS[0].model_fields["skip"] + assert report.description in reference + assert f"default ``{report.default!r}``" in reference + assert f"Example: ``{report.examples[0]!r}``." in reference + + +def test_spike_has_no_runtime_gui_yamview_or_sphinx_imports(): + imported = set() + for path in Path(__file__).parent.glob("*.py"): + if path.name.startswith("test_"): + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} + assert not any( + name == item or name.startswith(item + ".") + for name in imported + for item in forbidden + ) From 9b61dbe8c2624377c3a30b0ff53567e421daa82f Mon Sep 17 00:00:00 2001 From: alvaro Date: Thu, 13 Aug 2026 16:17:03 +0200 Subject: [PATCH 06/14] docs based on this new approach --- docs/source/_examples/recipe_v2.yml | 71 + .../_static/recipe_language.schema.json | 2085 +++++++++++++++++ docs/source/conf.py | 8 +- docs/source/dependency_license_analysis.rst | 5 +- docs/source/gui_event_handling.rst | 2 + docs/source/index.rst | 1 + docs/source/recipe_language_architecture.rst | 480 ++-- docs/source/recipe_language_reference.rst | 784 +++++++ docs/source/usage.rst | 8 +- docs/source/yaml_format.rst | 159 +- reuse.toml | 3 + spikes/recipe_pydantic/__init__.py | 43 +- spikes/recipe_pydantic/artifacts.py | 89 + spikes/recipe_pydantic/models.py | 6 + spikes/recipe_pydantic/parser.py | 8 + spikes/recipe_pydantic/reference.py | 295 ++- .../recipe_pydantic/test_recipe_pydantic.py | 22 +- src/pypts/recipe_reference.py | 9 +- tests/unit_tests/test_recipe_pydantic_docs.py | 151 ++ 19 files changed, 3801 insertions(+), 428 deletions(-) create mode 100644 docs/source/_examples/recipe_v2.yml create mode 100644 docs/source/_static/recipe_language.schema.json create mode 100644 docs/source/recipe_language_reference.rst create mode 100644 spikes/recipe_pydantic/artifacts.py create mode 100644 tests/unit_tests/test_recipe_pydantic_docs.py diff --git a/docs/source/_examples/recipe_v2.yml b/docs/source/_examples/recipe_v2.yml new file mode 100644 index 0000000..deb455b --- /dev/null +++ b/docs/source/_examples/recipe_v2.yml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: CC-BY-SA-4.0 +--- +name: Recipe language 2 documentation example +version: "1.0" +recipe_version: 2.0.0 +description: Demonstrate canonical version 2 syntax and typed mappings. +main_sequence: Main +continue_on_error: false +report: overwrite +report_name_include_serial: true +test_package: acceptance.tests +globals: + target: 12 + saved_result: null +--- +sequence_name: Main +description: Run a measurement and then the calibration sequence. +parameters: {} +outputs: {} +locals: + expected: 12 + measured: null +setup_steps: [] +steps: + - steptype: PythonModuleStep + step_name: Measure channels + description: Exercise every input mapping and representative outputs. + id: measure-channels + skip: false + critical: true + continue_on_error: false + action_type: method + module: measurements.py + method_name: measure + input_mapping: + channels: {type: direct, value: [0, 1], indexed: true} + expected: {type: local, local_name: expected} + target: {type: global, global_name: target} + transform: {type: method, value: normalize} + output_mapping: + passed: {type: passfail} + exact: {type: equals, value: 12} + bounded: {type: range, min: 10, max: 14} + measured: {type: local, local_name: measured} + saved: {type: global, global_name: saved_result} + chart: {type: image} + - steptype: SequenceStep + step_name: Calibrate + description: Run a nested sequence and use its aggregate verdict. + sequence: {type: internal, name: Calibration} + input_mapping: {} + output_mapping: + result: {type: passthrough} + stored: {type: local, local_name: measured} +teardown_steps: [] +--- +sequence_name: Calibration +description: Wait for the equipment to stabilize. +parameters: {} +outputs: {} +locals: {} +setup_steps: [] +steps: + - steptype: WaitStep + step_name: Stabilize + description: Wait before returning to the caller. + input_mapping: + wait_time: {type: direct, value: 1} + output_mapping: {} +teardown_steps: [] diff --git a/docs/source/_static/recipe_language.schema.json b/docs/source/_static/recipe_language.schema.json new file mode 100644 index 0000000..24382b7 --- /dev/null +++ b/docs/source/_static/recipe_language.schema.json @@ -0,0 +1,2085 @@ +{ + "$comment": "SPDX-FileCopyrightText: 2026 CERN ; SPDX-License-Identifier: CC-BY-SA-4.0", + "$defs": { + "DirectInput": { + "additionalProperties": false, + "description": "Provides a literal value.", + "properties": { + "indexed": { + "default": false, + "description": "Expand a list into indexed steps.", + "examples": [ + false + ], + "title": "Indexed", + "type": "boolean" + }, + "type": { + "const": "direct", + "description": "Input source type.", + "examples": [ + "direct" + ], + "title": "Type", + "type": "string" + }, + "value": { + "description": "Literal input value.", + "examples": [ + 1 + ], + "title": "Value" + } + }, + "required": [ + "type", + "value" + ], + "title": "DirectInput", + "type": "object" + }, + "EqualsOutput": { + "additionalProperties": false, + "description": "Passes when the output equals the configured value.", + "properties": { + "type": { + "const": "equals", + "description": "Output mapping type.", + "examples": [ + "equals" + ], + "title": "Type", + "type": "string" + }, + "value": { + "description": "Expected value.", + "examples": [ + 3 + ], + "title": "Value" + } + }, + "required": [ + "type", + "value" + ], + "title": "EqualsOutput", + "type": "object" + }, + "FileDestination": { + "additionalProperties": false, + "description": "Destination used by a file-loading step.", + "properties": { + "type": { + "description": "Variable scope.", + "enum": [ + "local", + "global" + ], + "examples": [ + "local" + ], + "title": "Type", + "type": "string" + }, + "variable": { + "description": "Destination variable name.", + "examples": [ + "selected_file" + ], + "title": "Variable", + "type": "string" + } + }, + "required": [ + "type", + "variable" + ], + "title": "FileDestination", + "type": "object" + }, + "GlobalInput": { + "additionalProperties": false, + "description": "Reads a recipe-global variable.", + "properties": { + "global_name": { + "description": "Global variable name.", + "examples": [ + "global_value" + ], + "title": "Global Name", + "type": "string" + }, + "type": { + "const": "global", + "description": "Input source type.", + "examples": [ + "global" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "global_name" + ], + "title": "GlobalInput", + "type": "object" + }, + "GlobalOutput": { + "additionalProperties": false, + "description": "Stores the output in a recipe-global variable.", + "properties": { + "global_name": { + "description": "Global destination variable.", + "examples": [ + "saved" + ], + "title": "Global Name", + "type": "string" + }, + "type": { + "const": "global", + "description": "Output mapping type.", + "examples": [ + "global" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "global_name" + ], + "title": "GlobalOutput", + "type": "object" + }, + "ImageOutput": { + "additionalProperties": false, + "description": "Publishes an image output for presentation.", + "properties": { + "type": { + "const": "image", + "description": "Output mapping type.", + "examples": [ + "image" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "ImageOutput", + "type": "object" + }, + "InputMapping": { + "discriminator": { + "mapping": { + "direct": "#/$defs/DirectInput", + "global": "#/$defs/GlobalInput", + "local": "#/$defs/LocalInput", + "method": "#/$defs/MethodInput" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/DirectInput" + }, + { + "$ref": "#/$defs/LocalInput" + }, + { + "$ref": "#/$defs/GlobalInput" + }, + { + "$ref": "#/$defs/MethodInput" + } + ] + }, + "InternalSequenceReference": { + "additionalProperties": false, + "description": "Reference to another sequence in this recipe.", + "properties": { + "name": { + "description": "Target sequence name.", + "examples": [ + "Calibration" + ], + "title": "Name", + "type": "string" + }, + "type": { + "const": "internal", + "description": "Reference kind.", + "examples": [ + "internal" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "name" + ], + "title": "InternalSequenceReference", + "type": "object" + }, + "LocalInput": { + "additionalProperties": false, + "description": "Reads a sequence-local variable.", + "properties": { + "local_name": { + "description": "Local variable name.", + "examples": [ + "local_value" + ], + "title": "Local Name", + "type": "string" + }, + "type": { + "const": "local", + "description": "Input source type.", + "examples": [ + "local" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "local_name" + ], + "title": "LocalInput", + "type": "object" + }, + "LocalOutput": { + "additionalProperties": false, + "description": "Stores the output in a sequence-local variable.", + "properties": { + "local_name": { + "description": "Local destination variable.", + "examples": [ + "saved" + ], + "title": "Local Name", + "type": "string" + }, + "type": { + "const": "local", + "description": "Output mapping type.", + "examples": [ + "local" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "local_name" + ], + "title": "LocalOutput", + "type": "object" + }, + "MethodInput": { + "additionalProperties": false, + "description": "Resolves a method reference for the step.", + "properties": { + "type": { + "const": "method", + "description": "Input source type.", + "examples": [ + "method" + ], + "title": "Type", + "type": "string" + }, + "value": { + "description": "Method reference.", + "examples": [ + "helper" + ], + "title": "Value" + } + }, + "required": [ + "type", + "value" + ], + "title": "MethodInput", + "type": "object" + }, + "OutputMapping": { + "discriminator": { + "mapping": { + "equals": "#/$defs/EqualsOutput", + "global": "#/$defs/GlobalOutput", + "image": "#/$defs/ImageOutput", + "local": "#/$defs/LocalOutput", + "passfail": "#/$defs/PassFailOutput", + "passthrough": "#/$defs/PassthroughOutput", + "range": "#/$defs/RangeOutput" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/PassFailOutput" + }, + { + "$ref": "#/$defs/EqualsOutput" + }, + { + "$ref": "#/$defs/RangeOutput" + }, + { + "$ref": "#/$defs/PassthroughOutput" + }, + { + "$ref": "#/$defs/LocalOutput" + }, + { + "$ref": "#/$defs/GlobalOutput" + }, + { + "$ref": "#/$defs/ImageOutput" + } + ] + }, + "PassFailOutput": { + "additionalProperties": false, + "description": "Interprets the output as a pass/fail verdict.", + "properties": { + "type": { + "const": "passfail", + "description": "Output mapping type.", + "examples": [ + "passfail" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "PassFailOutput", + "type": "object" + }, + "PassthroughOutput": { + "additionalProperties": false, + "description": "Uses the nested result without adding a verdict.", + "properties": { + "type": { + "const": "passthrough", + "description": "Output mapping type.", + "examples": [ + "passthrough" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "PassthroughOutput", + "type": "object" + }, + "PythonModuleStep": { + "additionalProperties": false, + "description": "Calls a method or reads/writes an attribute in a Python module.", + "properties": { + "action_type": { + "description": "Operation performed on the Python module.", + "enum": [ + "method", + "read_attribute", + "write_attribute" + ], + "examples": [ + "method" + ], + "title": "Action Type", + "type": "string" + }, + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "method_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Method name for method actions.", + "examples": [ + "run" + ], + "title": "Method Name" + }, + "module": { + "description": "Python module path.", + "examples": [ + "tests.py" + ], + "title": "Module", + "type": "string" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "PythonModuleStep", + "description": "Canonical registered step type.", + "examples": [ + "PythonModuleStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype", + "action_type", + "module" + ], + "title": "PythonModuleStep", + "type": "object" + }, + "RangeOutput": { + "additionalProperties": false, + "description": "Passes when the output is within an inclusive range.", + "properties": { + "max": { + "description": "Maximum accepted value.", + "examples": [ + 4 + ], + "title": "Max" + }, + "min": { + "description": "Minimum accepted value.", + "examples": [ + 1 + ], + "title": "Min" + }, + "type": { + "const": "range", + "description": "Output mapping type.", + "examples": [ + "range" + ], + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "min", + "max" + ], + "title": "RangeOutput", + "type": "object" + }, + "RecipeHeader": { + "additionalProperties": false, + "description": "The first YAML document, identifying a recipe and its entry sequence.", + "properties": { + "continue_on_error": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Recipe-wide error policy.", + "examples": [ + false + ], + "title": "Continue On Error" + }, + "description": { + "description": "Purpose of the recipe.", + "examples": [ + "Acceptance tests." + ], + "title": "Description", + "type": "string" + }, + "globals": { + "additionalProperties": true, + "description": "Recipe-wide variables.", + "examples": [ + {} + ], + "title": "Globals", + "type": "object" + }, + "main_sequence": { + "description": "Sequence where execution begins.", + "examples": [ + "Main" + ], + "title": "Main Sequence", + "type": "string" + }, + "name": { + "description": "Human-readable recipe name.", + "examples": [ + "Hardware acceptance" + ], + "title": "Name", + "type": "string" + }, + "recipe_version": { + "const": "2.0.0", + "description": "Version of the recipe language contract.", + "examples": [ + "2.0.0" + ], + "title": "Recipe Version", + "type": "string" + }, + "report": { + "default": "overwrite", + "description": "Report file mode.", + "enum": [ + "overwrite", + "append" + ], + "examples": [ + "overwrite" + ], + "title": "Report", + "type": "string" + }, + "report_name_include_serial": { + "default": false, + "description": "Include the serial number in the report name.", + "examples": [ + false + ], + "title": "Report Name Include Serial", + "type": "boolean" + }, + "test_package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Package containing recipe test modules.", + "examples": [ + "acceptance" + ], + "title": "Test Package" + }, + "version": { + "description": "Version of this recipe.", + "examples": [ + "1.0" + ], + "title": "Version", + "type": "string" + } + }, + "required": [ + "name", + "version", + "recipe_version", + "description", + "main_sequence", + "globals" + ], + "title": "RecipeHeader", + "type": "object" + }, + "SSHCloseStep": { + "additionalProperties": false, + "description": "Closes the SSH client stored in recipe globals.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "SSHCloseStep", + "description": "Canonical registered step type.", + "examples": [ + "SSHCloseStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "SSHCloseStep", + "type": "object" + }, + "SSHConnectStep": { + "additionalProperties": false, + "description": "Opens the SSH client stored in recipe globals.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "SSHConnectStep", + "description": "Canonical registered step type.", + "examples": [ + "SSHConnectStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "SSHConnectStep", + "type": "object" + }, + "SSHUploadStep": { + "additionalProperties": false, + "description": "Uploads files through an SSH connection.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "files": { + "description": "Local and remote file pairs to upload.", + "examples": [ + [ + { + "local": "bin/tool", + "remote": "/tmp/tool" + } + ] + ], + "items": { + "$ref": "#/$defs/UploadFile" + }, + "title": "Files", + "type": "array" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "local_package": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional package containing local resources.", + "examples": [ + "my_package" + ], + "title": "Local Package" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "permissions": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional remote permissions.", + "examples": [ + "0755" + ], + "title": "Permissions" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "skip_if_sha256_match": { + "default": false, + "description": "Skip files whose remote checksum matches.", + "examples": [ + false + ], + "title": "Skip If Sha256 Match", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "SSHUploadStep", + "description": "Canonical registered step type.", + "examples": [ + "SSHUploadStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype", + "files" + ], + "title": "SSHUploadStep", + "type": "object" + }, + "Sequence": { + "additionalProperties": false, + "description": "One named executable sequence document.", + "properties": { + "description": { + "description": "Purpose of the sequence.", + "examples": [ + "Main sequence." + ], + "title": "Description", + "type": "string" + }, + "locals": { + "additionalProperties": true, + "description": "Variables local to the sequence.", + "examples": [ + {} + ], + "title": "Locals", + "type": "object" + }, + "outputs": { + "additionalProperties": true, + "description": "Reserved sequence output metadata.", + "examples": [ + {} + ], + "title": "Outputs", + "type": "object" + }, + "parameters": { + "additionalProperties": true, + "description": "Reserved sequence input metadata.", + "examples": [ + {} + ], + "title": "Parameters", + "type": "object" + }, + "sequence_name": { + "description": "Unique sequence name.", + "examples": [ + "Main" + ], + "title": "Sequence Name", + "type": "string" + }, + "setup_steps": { + "description": "Steps run before the main steps.", + "examples": [ + [] + ], + "items": { + "$ref": "#/$defs/Step" + }, + "title": "Setup Steps", + "type": "array" + }, + "steps": { + "description": "Ordered main steps.", + "examples": [ + [] + ], + "items": { + "$ref": "#/$defs/Step" + }, + "title": "Steps", + "type": "array" + }, + "teardown_steps": { + "description": "Steps run during teardown.", + "examples": [ + [] + ], + "items": { + "$ref": "#/$defs/Step" + }, + "title": "Teardown Steps", + "type": "array" + } + }, + "required": [ + "sequence_name", + "description", + "parameters", + "outputs", + "locals", + "setup_steps", + "steps", + "teardown_steps" + ], + "title": "Sequence", + "type": "object" + }, + "SequenceStep": { + "additionalProperties": false, + "description": "Runs another sequence as a step.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "sequence": { + "$ref": "#/$defs/InternalSequenceReference", + "description": "Internal sequence reference.", + "examples": [ + { + "name": "Calibration", + "type": "internal" + } + ] + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "SequenceStep", + "description": "Canonical registered step type.", + "examples": [ + "SequenceStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype", + "sequence" + ], + "title": "SequenceStep", + "type": "object" + }, + "SerialNumberStep": { + "additionalProperties": false, + "description": "Captures the device serial number.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "SerialNumberStep", + "description": "Canonical registered step type.", + "examples": [ + "SerialNumberStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "SerialNumberStep", + "type": "object" + }, + "Step": { + "discriminator": { + "mapping": { + "PythonModuleStep": "#/$defs/PythonModuleStep", + "SSHCloseStep": "#/$defs/SSHCloseStep", + "SSHConnectStep": "#/$defs/SSHConnectStep", + "SSHUploadStep": "#/$defs/SSHUploadStep", + "SequenceStep": "#/$defs/SequenceStep", + "SerialNumberStep": "#/$defs/SerialNumberStep", + "UserInteractionStep": "#/$defs/UserInteractionStep", + "UserLoadingStep": "#/$defs/UserLoadingStep", + "UserRunMethodStep": "#/$defs/UserRunMethodStep", + "UserWriteStep": "#/$defs/UserWriteStep", + "WaitStep": "#/$defs/WaitStep" + }, + "propertyName": "steptype" + }, + "oneOf": [ + { + "$ref": "#/$defs/PythonModuleStep" + }, + { + "$ref": "#/$defs/SequenceStep" + }, + { + "$ref": "#/$defs/UserInteractionStep" + }, + { + "$ref": "#/$defs/WaitStep" + }, + { + "$ref": "#/$defs/UserLoadingStep" + }, + { + "$ref": "#/$defs/UserRunMethodStep" + }, + { + "$ref": "#/$defs/UserWriteStep" + }, + { + "$ref": "#/$defs/SerialNumberStep" + }, + { + "$ref": "#/$defs/SSHConnectStep" + }, + { + "$ref": "#/$defs/SSHCloseStep" + }, + { + "$ref": "#/$defs/SSHUploadStep" + } + ] + }, + "UploadFile": { + "additionalProperties": false, + "description": "One local-to-remote SSH upload pair.", + "properties": { + "local": { + "description": "Local file or package resource.", + "examples": [ + "bin/tool" + ], + "title": "Local", + "type": "string" + }, + "remote": { + "description": "Remote destination path.", + "examples": [ + "/tmp/tool" + ], + "title": "Remote", + "type": "string" + } + }, + "required": [ + "local", + "remote" + ], + "title": "UploadFile", + "type": "object" + }, + "UserInteractionStep": { + "additionalProperties": false, + "description": "Displays an operator interaction prompt.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "UserInteractionStep", + "description": "Canonical registered step type.", + "examples": [ + "UserInteractionStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "UserInteractionStep", + "type": "object" + }, + "UserLoadingStep": { + "additionalProperties": false, + "description": "Prompts the operator to select a file.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "file_save_location": { + "anyOf": [ + { + "$ref": "#/$defs/FileDestination" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Local or global destination for the selected file.", + "examples": [ + { + "type": "local", + "variable": "selected_file" + } + ] + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "UserLoadingStep", + "description": "Canonical registered step type.", + "examples": [ + "UserLoadingStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "UserLoadingStep", + "type": "object" + }, + "UserRunMethodStep": { + "additionalProperties": false, + "description": "Optionally runs a Python method after an operator response.", + "properties": { + "action_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Python action type.", + "examples": [ + "method" + ], + "title": "Action Type" + }, + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "method_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Python method name.", + "examples": [ + "run" + ], + "title": "Method Name" + }, + "module": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional Python module path.", + "examples": [ + "tests.py" + ], + "title": "Module" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "UserRunMethodStep", + "description": "Canonical registered step type.", + "examples": [ + "UserRunMethodStep" + ], + "title": "Steptype", + "type": "string" + }, + "trigger_response": { + "anyOf": [ + { + "type": "string" + }, + { + "items": {}, + "type": "array" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Operator response that triggers execution.", + "examples": [ + "run" + ], + "title": "Trigger Response" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "UserRunMethodStep", + "type": "object" + }, + "UserWriteStep": { + "additionalProperties": false, + "description": "Writes an operator-provided value to a configured destination.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "UserWriteStep", + "description": "Canonical registered step type.", + "examples": [ + "UserWriteStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "UserWriteStep", + "type": "object" + }, + "WaitStep": { + "additionalProperties": false, + "description": "Waits for a non-negative duration in seconds.", + "properties": { + "continue_on_error": { + "default": false, + "description": "Per-step error policy.", + "examples": [ + false + ], + "title": "Continue On Error", + "type": "boolean" + }, + "critical": { + "default": false, + "description": "Stop on error when policy permits continuation.", + "examples": [ + false + ], + "title": "Critical", + "type": "boolean" + }, + "description": { + "description": "Purpose of the step.", + "examples": [ + "Run a test operation." + ], + "title": "Description", + "type": "string" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional stable step identifier.", + "examples": [ + "test-1" + ], + "title": "Id" + }, + "input_mapping": { + "additionalProperties": { + "$ref": "#/$defs/InputMapping" + }, + "description": "Named input sources.", + "examples": [ + {} + ], + "title": "Input Mapping", + "type": "object" + }, + "output_mapping": { + "additionalProperties": { + "$ref": "#/$defs/OutputMapping" + }, + "description": "Named verdicts and destinations.", + "examples": [ + {} + ], + "title": "Output Mapping", + "type": "object" + }, + "skip": { + "default": false, + "description": "Skip execution.", + "examples": [ + false + ], + "title": "Skip", + "type": "boolean" + }, + "step_name": { + "description": "Human-readable step name.", + "examples": [ + "Run test" + ], + "title": "Step Name", + "type": "string" + }, + "steptype": { + "const": "WaitStep", + "description": "Canonical registered step type.", + "examples": [ + "WaitStep" + ], + "title": "Steptype", + "type": "string" + } + }, + "required": [ + "step_name", + "description", + "steptype" + ], + "title": "WaitStep", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Aggregate typed recipe used by tooling and JSON Schema consumers.", + "properties": { + "header": { + "$ref": "#/$defs/RecipeHeader", + "description": "Recipe header document." + }, + "sequences": { + "description": "Sequence documents.", + "items": { + "$ref": "#/$defs/Sequence" + }, + "minItems": 1, + "title": "Sequences", + "type": "array" + } + }, + "required": [ + "header", + "sequences" + ], + "title": "Recipe", + "type": "object" +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 776c000..4d52a21 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import datetime +import importlib.util from pypts._version import __version__ @@ -20,12 +21,13 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'acc_py_sphinx.theme', 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.napoleon', ] +if importlib.util.find_spec("acc_py_sphinx") is not None: + extensions.insert(0, "acc_py_sphinx.theme") # Add any paths that contain templates here, relative to this directory. @@ -42,12 +44,12 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "acc_py" +html_theme = "acc_py" if "acc_py_sphinx.theme" in extensions else "alabaster" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ["_static"] +html_static_path = ["_static"] html_show_sphinx = False html_show_sourcelink = True diff --git a/docs/source/dependency_license_analysis.rst b/docs/source/dependency_license_analysis.rst index 5684237..acbaa07 100644 --- a/docs/source/dependency_license_analysis.rst +++ b/docs/source/dependency_license_analysis.rst @@ -102,8 +102,6 @@ nidmm (1.4.8) :Source: nimi-python GitHub repository :Compatibility: ✅ **Full compatibility** - MIT licensed National Instruments drivers -<<<<<<< Updated upstream -======= pymeasure (0.15.0) ~~~~~~~~~~~~~~~~~~ :License: MIT @@ -113,7 +111,6 @@ pymeasure (0.15.0) libraries without restriction. The pymeasure package itself is not affected by pypts' LGPL licence; end users may use pymeasure under its own MIT terms independently. ->>>>>>> Stashed changes nptdms ~~~~~~ :License: LGPL (GNU Library or Lesser General Public License) @@ -361,4 +358,4 @@ This analysis was conducted using: - Web searches for license verification **Last Updated:** June 2025 -**Next Review:** Recommended annually or when adding new dependencies \ No newline at end of file +**Next Review:** Recommended annually or when adding new dependencies diff --git a/docs/source/gui_event_handling.rst b/docs/source/gui_event_handling.rst index a1bece1..70c4259 100644 --- a/docs/source/gui_event_handling.rst +++ b/docs/source/gui_event_handling.rst @@ -56,6 +56,7 @@ recipe progress and results: Crucially, it also stores the unique `step.id` (UUID) associated with each step name using `Qt.ItemDataRole.UserRole`. * **Update Mechanism (ViewModel Pattern):** + 1. When a step is about to run, `recipe.Runtime` puts a `pre_run_step` event onto the `event_queue` containing the `recipe.Step` object. 2. `RecipeEventProxy` receives this, creates a ViewModel `{'step_uuid': ..., 'step_name': ...}`, @@ -104,6 +105,7 @@ recipe progress and results: of the entire recipe *after* it has finished execution. * **Initialization:** This view is populated only once when the recipe finishes. * **Update Mechanism (ViewModel Dictionary + Coupled Model):** + 1. When the recipe finishes, `recipe.Runtime` puts a `post_run_recipe` event onto the `event_queue` containing the final `List[recipe.StepResult]`. 2. `RecipeEventProxy` receives this event. diff --git a/docs/source/index.rst b/docs/source/index.rst index 1a361ae..3ba2de7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,6 +57,7 @@ Documentation contents api architecture recipe_language_architecture + recipe_language_reference gui_architecture yaml_format instruments diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index 32e176b..45d093b 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -2,221 +2,265 @@ .. .. SPDX-License-Identifier: CC-BY-SA-4.0 -Recipe Language Architecture -============================ - -This page describes the recipe language model, the isolated parser, and the -rules for evolving them. The parser is currently independent of recipe -execution, ``verify_recipe``, YamVIEW, and Sphinx reference publication. Those -runtime and UI consumers will move onto the shared model in later integration -work. - -Design goals ------------- - -The recipe language has one contract and one parsing path. YAML loading, -language validation, normalized data, and runtime construction are separate -responsibilities. In particular, parsing a recipe must never import or invoke -the runtime, steps, GUI, or documentation machinery. - -The current architecture has two source modules: - -``pypts.recipe_language`` - Defines the framework-independent contract. Its field and step - specifications describe accepted recipe documents, while - ``validate_recipe_documents`` validates already-loaded Python values. It - has no YAML or runtime dependency. - -``pypts.recipe_parser`` - Safely loads YAML, records source locations, delegates language rules to the - contract, and constructs immutable typed definitions. It also serializes a - typed recipe into canonical YAML. - -``pypts.recipe_reference`` - Validates the registered examples and generates the deterministic standalone - RST language reference. The generated artifact remains outside the Sphinx - source tree until framework integration is accepted. - -The intended flow is:: - - recipe YAML - | - v - safe YAML loading and source indexing - | - v - recipe_language contract validation - | - +---- errors and warnings with source spans - | - v - immutable RecipeDefinition - | - +---- dump_recipe() -> canonical recipe YAML - | - +---- generated reference, and future runtime and GUI consumers - -Parser operation ----------------- - -``parse_recipe_text`` and ``parse_recipe_file`` return a ``ParseResult``. The -parser performs these stages in order: - -#. Reject non-text, empty, or unreadable input. -#. Compose the YAML with ``SafeLoader`` to index nodes and their one-based line - and column positions. Duplicate mapping keys and recursive aliases are - diagnosed here. -#. Safely construct the YAML documents. Malformed YAML, unsafe tags, and - construction failures become diagnostics rather than runtime objects. -#. Pass the loaded documents to ``validate_recipe_documents``. This is where - header, sequence, step, mapping, reference, and lifecycle rules are applied. -#. Attach the closest available source span to each contract diagnostic and - emit warnings for accepted legacy spellings or implicit forms. -#. If any error exists, return no recipe. Otherwise, normalize the documents - into an immutable ``RecipeDefinition``. - -``ParseResult.errors`` and ``ParseResult.warnings`` split diagnostics by -severity. ``ParseResult.require_recipe()`` returns the model on success and -raises ``RecipeParseError`` with all diagnostics on failure. Callers that need -to present every issue should inspect the result before requiring the model. - -Diagnostics contain a stable code, message, semantic path, severity, source -name, and optional ``SourceSpan``. Consumers should branch on the code rather -than matching message text. A source span is half-open; its line and column -values are one-based and its character offset is zero-based. - -Typed and normalized model --------------------------- - -The model is made of frozen definitions for the recipe header, sequences, -steps, and each input and output mapping variant. Arbitrary mappings that are -part of recipe data use the immutable, insertion-ordered ``FrozenMap``. Source -spans and source names do not participate in semantic equality, which makes a -parse/dump/reparse comparison independent of file location. - -Normalization currently includes: - -* canonical step type casing; -* explicit typed input and output definitions; -* false defaults for step flags; -* empty mappings for optional mapping fields; -* recipe report defaults; and -* removal of runtime-ignored legacy sequence metadata from the typed model. - -``dump_recipe`` emits stable, explicit-start, multi-document YAML. It writes -canonical step names, explicit input types, explicit defaults, and stable field -ordering. It does not preserve comments or the source's original formatting. -The semantic guarantee is therefore model round-trip equality, not textual -round-trip equality. - -Using the parser ----------------- - -Parse in-memory YAML when the caller already owns the text: - -.. code-block:: python - - from pypts.recipe_parser import parse_recipe_text - - result = parse_recipe_text(source, source_name="recipe.yml") - if result.errors: - for diagnostic in result.errors: - print(diagnostic.code, diagnostic.path, diagnostic.span) - else: - recipe = result.require_recipe() - -Use ``parse_recipe_file`` when the parser should read the file and report I/O -or decoding failures as diagnostics. Use ``dump_recipe`` only with a valid -``RecipeDefinition``. - -Architecture boundaries ------------------------ - -Keep the dependency direction narrow: - -* ``recipe_language`` must not depend on YAML, runtime classes, concrete steps, - YamVIEW, or Sphinx. -* ``recipe_parser`` may depend on PyYAML and ``recipe_language`` but not on - runtime, concrete steps, YamVIEW, or Sphinx. -* Runtime and UI adapters may consume parser models after integration; parser - models must not consume those adapters. -* Syntax reference fields, constraints, and examples are generated and checked - from the language specifications. Architecture prose explains - responsibilities and extension workflows rather than duplicating field - tables. - -These rules keep syntax inspection safe and make the parser usable by command -line tools, editors, the GUI, tests, and documentation without constructing -hardware-facing runtime objects. - -Maintaining the language ------------------------- - -Adding or changing a step -~~~~~~~~~~~~~~~~~~~~~~~~ - -#. Update its ``StepSpec`` and ``FieldSpec`` entries in - ``pypts.recipe_language``. Do not create a second field list in a consumer. -#. Add semantic checks beside the shared contract validation when a constraint - cannot be represented by required fields and value types. -#. Add a valid executable parser fixture and focused invalid cases for every - new constraint. -#. Confirm canonical serialization contains the step-specific configuration in - specification order and parse/dump/reparse preserves the model. -#. During framework integration, update only the adapter that constructs the - runtime step from ``StepDefinition``. -#. Regenerate and check the standalone syntax reference:: - - python -m pypts.recipe_reference docs/generated/recipe_language_reference.rst - python -m pypts.recipe_reference --check docs/generated/recipe_language_reference.rst - -Changing input or output mappings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -#. Update the allowed and required fields in the contract validator. -#. Add or adjust the corresponding frozen mapping definition, model builder, - and serializer branch in ``recipe_parser``. -#. Test accepted values, every relevant failure, source spans, normalization, - and canonical round trips. -#. Check runtime and GUI adapters after integration. They should dispatch on - typed mapping definitions instead of maintaining supported-type lists. - -Evolving ``recipe_version`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Recipe format changes must be explicit. Do not silently reinterpret an -existing version. First describe compatibility and migration behavior, then -add version-specific contract handling and fixtures. Preserve parsing for a -supported old version or emit a precise unsupported-version diagnostic. A -canonical dump must state the version whose semantics it writes. - -Retiring legacy syntax -~~~~~~~~~~~~~~~~~~~~~~ - -Legacy syntax should move through an observable sequence: accept and normalize -with a stable warning, document the canonical replacement, measure and migrate -the bundled examples and consumers, and only then reject it in a declared -recipe version. Never remove an accepted form merely by changing a runtime -constructor. - -Verification ------------- - -The parser and language tests are in ``tests/unit_tests/test_recipe_parser.py`` -and ``tests/unit_tests/test_recipe_language.py``. Run them directly while -editing the contract, then run the complete suite: - -.. code-block:: console - - python -m pytest tests/unit_tests/test_recipe_language.py tests/unit_tests/test_recipe_parser.py - python -m pytest - -The acceptance corpus consists of every non-empty bundled recipe. Each must -parse, dump, reparse, and compare equal as a model. The comment-only draft is -intentionally invalid. Isolation tests also protect the parser from importing -runtime, step, GUI, or Sphinx modules. - -After the integration phase, verification must also cover runtime construction -equivalence, GUI-produced canonical YAML, successful Sphinx builds with -warnings treated as errors, and the absence of duplicated consumer-side field -or type registries. +Recipe Language 2 Architecture +============================== + +.. important:: + + This page describes the accepted architecture for recipe language + ``2.0.0``. The Pydantic implementation is currently an isolated reference + under ``spikes/recipe_pydantic``. Production parsing, execution, YamVIEW, + and bundled recipes continue to use the version 1 implementation until the + integration phase is complete. See :doc:`yaml_format` for current runtime + guidance. + +Exact version 2 fields, types, defaults, and examples are in the generated +:doc:`recipe_language_reference`. The aggregate schema is also available as +:download:`recipe_language.schema.json +<_static/recipe_language.schema.json>`. + +Design and dependency direction +------------------------------- + +The version 2 design has one structural definition. Strict, frozen Pydantic +models own field names, types, required status, defaults, descriptions, +examples, aliases, discriminators, serialization metadata, and JSON Schema. +Consumers inspect either the typed model or its generated schema; they do not +maintain another field registry. + +The modules and artifacts have deliberately one-way dependencies:: + + models.py + | Pydantic fields and discriminated unions + +--------------------------+ + | | + v v + parser.py JSON Schema generator + | | + | v + | recipe_language.schema.json + | | + | v + | JSON-only RST renderer + | | + v v + typed Recipe recipe_language_reference.rst + | | + v v + future runtime adapter Sphinx + +``models.py`` never imports YAML, runtime classes, concrete steps, YamVIEW, or +Sphinx. ``parser.py`` depends on the models and PyYAML, but still does not +import runtime or UI code. Documentation generation reads the model only to +create JSON Schema; the RST renderer reads the committed JSON file and has no +Pydantic or Sphinx dependency. + +Parsing and information flow +---------------------------- + +Pydantic validates already-constructed Python values; it is not a YAML parser. +Safe loading, source positions, structural validation, and application +semantics therefore remain separate stages:: + + recipe YAML text/file + | + v + SafeLoader composition --------> YAML node/path/source-span index + | | + +--> duplicate key / alias checks | + | | + v | + safe Python documents | + | | + v | + strict Pydantic models | + | | + v | + cross-document semantic pass | + | | + +--> diagnostics <-----------------+ + | code, path, severity, source, span + v + frozen aggregate Recipe + | + +--> canonical multi-document YAML + | + +--> future sequencer/runtime adapter + +``parse_recipe_text`` and ``parse_recipe_file`` return ``ParseResult``. A +valid result owns an aggregate :ref:`recipe-v2-header` plus one or more +:ref:`recipe-v2-sequence` models. ``require_recipe()`` raises with the complete +diagnostic tuple when errors exist. ``dump_recipe`` writes canonical version 2 +YAML; comments and original formatting are not a round-trip guarantee. + +Structural and custom semantic rules +------------------------------------ + +Rules local to one model stay beside that model. Pydantic reports them with a +precise nested location, which the parser translates to the PyPTS diagnostic +envelope and nearest YAML span. + +For example, an indexed :ref:`recipe-v2-input-direct` must hold a list: + +.. literalinclude:: ../../spikes/recipe_pydantic/models.py + :language: python + :start-after: # docs:indexed-direct-start + :end-before: # docs:indexed-direct-end + :dedent: 4 + +A Python method action requires ``method_name``: + +.. literalinclude:: ../../spikes/recipe_pydantic/models.py + :language: python + :start-after: # docs:method-name-start + :end-before: # docs:method-name-end + :dedent: 4 + +Likewise, :ref:`recipe-v2-step-waitstep` requires a named ``wait_time`` input: + +.. literalinclude:: ../../spikes/recipe_pydantic/models.py + :language: python + :start-after: # docs:wait-time-start + :end-before: # docs:wait-time-end + :dedent: 4 + +Other rules require context that no individual JSON object or JSON Schema can +see. They remain in one explicit semantic pass. Sequence names must be +unique and ``main_sequence`` must resolve: + +.. literalinclude:: ../../spikes/recipe_pydantic/parser.py + :language: python + :start-after: # docs:sequence-semantics-start + :end-before: # docs:sequence-semantics-end + :dedent: 4 + +Every :ref:`recipe-v2-step-sequencestep` target is then resolved across all +loaded documents: + +.. literalinclude:: ../../spikes/recipe_pydantic/parser.py + :language: python + :start-after: # docs:nested-reference-start + :end-before: # docs:nested-reference-end + :dedent: 12 + +Indexed lists on one step must have equal lengths, while +:ref:`recipe-v2-output-passthrough` must be the only verdict-producing output: + +.. literalinclude:: ../../spikes/recipe_pydantic/parser.py + :language: python + :start-after: # docs:mapping-semantics-start + :end-before: # docs:mapping-semantics-end + :dedent: 12 + +SSH rules need both recipe globals and execution order. The semantic pass +checks required connection globals and credentials, rejects an upload before a +connection, and requires an opened connection to be closed: + +.. literalinclude:: ../../spikes/recipe_pydantic/parser.py + :language: python + :start-after: # docs:ssh-semantics-start + :end-before: # docs:ssh-semantics-end + :dedent: 8 + +These rules are documented manually because JSON Schema describes the +aggregate structure, not multi-document YAML safety, source spans, equality +between sibling list lengths, reference resolution, or ordered lifecycle +state. + +How YamVIEW will consume the language +-------------------------------------- + +YamVIEW will treat the aggregate JSON Schema as its form description. The +``Step``, ``InputMapping``, and ``OutputMapping`` discriminator maps enumerate +available variants; referenced definitions provide properties, required +fields, strict types, defaults, descriptions, examples, and allowed literal +values. + +The intended editor flow is:: + + committed JSON Schema + | + +--> discriminator choices --> step/mapping selectors + | + +--> referenced properties --> labels, controls, help, defaults + | + v + edited aggregate document --> canonical YAML text + | + v + parse_recipe_text() + | | + | +--> diagnostics and source spans + v + typed Recipe + +Local widget code may choose a suitable control for a JSON type, but it must +not own supported step names or field rules. Whole-recipe validation always +goes through the parser so semantic rules and YAML diagnostics are identical +between YamVIEW, command-line tools, and runtime loading. + +How the sequencer will consume the model +---------------------------------------- + +The sequencer integration begins only after parsing succeeds. It receives the +aggregate typed model, not raw YAML or loosely typed dictionaries:: + + ParseResult.require_recipe() + | + v + frozen Recipe model + | + v + runtime adapter + | | | + | | +--> typed input/output mapping adapters + | +-------> concrete runtime step construction + +------------> sequence table and nested reference binding + | + v + setup_steps -> steps -> teardown_steps + +The adapter will translate language models to runtime objects exactly once. +It must not reparse YAML, repeat structural validation, or keep another +supported-type registry. Invalid recipes never instantiate concrete runtime +steps. Runtime-only behavior—execution events, error policy, reports, hardware +access, and GUI interaction—stays downstream of the language model. + +Canonical documentation recipe +------------------------------ + +This documentation-owned fixture demonstrates the version 2 header, two +sequences, nested execution, canonical step names, explicit discriminators, +indexed input, every input variant, and representative verdict, storage, +image, and passthrough outputs. Tests validate and round-trip it with the +spike. It is not a production bundled recipe. + +.. literalinclude:: _examples/recipe_v2.yml + :language: yaml + :caption: Canonical recipe language 2 example + +Maintaining the documentation +----------------------------- + +The tracked artifacts have final paths under ``docs/source``. A normal Sphinx +build reads them directly and performs no generation or copying:: + + python -m spikes.recipe_pydantic.artifacts + +The command writes ``_static/recipe_language.schema.json`` from Pydantic and +then writes ``recipe_language_reference.rst`` by parsing that JSON. Check that +both committed files are current without modifying them:: + + python -m spikes.recipe_pydantic.artifacts --check + +When adding a step or mapping, update its Pydantic model and discriminated +union, add an independent round-trip fixture, regenerate both artifacts, and +review the schema and RST diffs. Add custom semantic code only when a rule +requires document, sibling, or ordering context. Handwritten architecture +prose explains those relationships; it must link to generated fields rather +than restating field tables. + +The documentation contract is protected by tests that compare the model to the +committed JSON, compare the JSON-only renderer to the committed RST, count all +discriminator variants, validate the example, check literal-include markers, +and build Sphinx with warnings treated as errors. diff --git a/docs/source/recipe_language_reference.rst b/docs/source/recipe_language_reference.rst new file mode 100644 index 0000000..ac7f8b5 --- /dev/null +++ b/docs/source/recipe_language_reference.rst @@ -0,0 +1,784 @@ +.. SPDX-FileCopyrightText: 2026 CERN +.. +.. SPDX-License-Identifier: CC-BY-SA-4.0 +.. +.. Generated from recipe_language.schema.json. Do not edit manually. + +Recipe Language 2.0 Reference +============================= + +This page is generated from the tracked aggregate JSON Schema. It describes +the accepted future recipe language model; production execution still uses +the version 1 language until Phase 6 integration is complete. + +:download:`Download the JSON Schema <_static/recipe_language.schema.json>`. + +See :doc:`recipe_language_architecture` for parsing, semantic rules, +documentation maintenance, and the planned YamVIEW and sequencer flows. + +Documents +--------- + +.. _recipe-v2-header: + +RecipeHeader +~~~~~~~~~~~~ + +The first YAML document, identifying a recipe and its entry sequence. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``continue_on_error`` + - ``bool | None`` + - optional; default ``null`` + - Recipe-wide error policy. Example: ``false``. + * - ``description`` + - ``str`` + - required + - Purpose of the recipe. Example: ``"Acceptance tests."``. + * - ``globals`` + - ``object`` + - required + - Recipe-wide variables. Example: ``{}``. + * - ``main_sequence`` + - ``str`` + - required + - Sequence where execution begins. Example: ``"Main"``. + * - ``name`` + - ``str`` + - required + - Human-readable recipe name. Example: ``"Hardware acceptance"``. + * - ``recipe_version`` + - ``'2.0.0'`` + - required + - Version of the recipe language contract. Example: ``"2.0.0"``. + * - ``report`` + - ``'overwrite' | 'append'`` + - optional; default ``"overwrite"`` + - Report file mode. Example: ``"overwrite"``. + * - ``report_name_include_serial`` + - ``bool`` + - optional; default ``false`` + - Include the serial number in the report name. Example: ``false``. + * - ``test_package`` + - ``str | None`` + - optional; default ``null`` + - Package containing recipe test modules. Example: ``"acceptance"``. + * - ``version`` + - ``str`` + - required + - Version of this recipe. Example: ``"1.0"``. + +.. _recipe-v2-sequence: + +Sequence +~~~~~~~~ + +One named executable sequence document. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``description`` + - ``str`` + - required + - Purpose of the sequence. Example: ``"Main sequence."``. + * - ``locals`` + - ``object`` + - required + - Variables local to the sequence. Example: ``{}``. + * - ``outputs`` + - ``object`` + - required + - Reserved sequence output metadata. Example: ``{}``. + * - ``parameters`` + - ``object`` + - required + - Reserved sequence input metadata. Example: ``{}``. + * - ``sequence_name`` + - ``str`` + - required + - Unique sequence name. Example: ``"Main"``. + * - ``setup_steps`` + - ``list[Step]`` + - required + - Steps run before the main steps. Example: ``[]``. + * - ``steps`` + - ``list[Step]`` + - required + - Ordered main steps. Example: ``[]``. + * - ``teardown_steps`` + - ``list[Step]`` + - required + - Steps run during teardown. Example: ``[]``. + +Nested structures +----------------- + +.. _recipe-v2-structure-internalsequencereference: + +InternalSequenceReference +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Reference to another sequence in this recipe. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``name`` + - ``str`` + - required + - Target sequence name. Example: ``"Calibration"``. + * - ``type`` + - ``'internal'`` + - required + - Reference kind. Example: ``"internal"``. + +.. _recipe-v2-structure-filedestination: + +FileDestination +~~~~~~~~~~~~~~~ + +Destination used by a file-loading step. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'local' | 'global'`` + - required + - Variable scope. Example: ``"local"``. + * - ``variable`` + - ``str`` + - required + - Destination variable name. Example: ``"selected_file"``. + +.. _recipe-v2-structure-uploadfile: + +UploadFile +~~~~~~~~~~ + +One local-to-remote SSH upload pair. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``local`` + - ``str`` + - required + - Local file or package resource. Example: ``"bin/tool"``. + * - ``remote`` + - ``str`` + - required + - Remote destination path. Example: ``"/tmp/tool"``. + +Common step fields +------------------ + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``continue_on_error`` + - ``bool`` + - optional; default ``false`` + - Per-step error policy. Example: ``false``. + * - ``critical`` + - ``bool`` + - optional; default ``false`` + - Stop on error when policy permits continuation. Example: ``false``. + * - ``description`` + - ``str`` + - required + - Purpose of the step. Example: ``"Run a test operation."``. + * - ``id`` + - ``str | None`` + - optional; default ``null`` + - Optional stable step identifier. Example: ``"test-1"``. + * - ``input_mapping`` + - ``dict[str, InputMapping]`` + - optional + - Named input sources. Example: ``{}``. + * - ``output_mapping`` + - ``dict[str, OutputMapping]`` + - optional + - Named verdicts and destinations. Example: ``{}``. + * - ``skip`` + - ``bool`` + - optional; default ``false`` + - Skip execution. Example: ``false``. + * - ``step_name`` + - ``str`` + - required + - Human-readable step name. Example: ``"Run test"``. + +Authorable steps +---------------- + +.. _recipe-v2-step-pythonmodulestep: + +PythonModuleStep +~~~~~~~~~~~~~~~~ + +Calls a method or reads/writes an attribute in a Python module. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``action_type`` + - ``'method' | 'read_attribute' | 'write_attribute'`` + - required + - Operation performed on the Python module. Example: ``"method"``. + * - ``method_name`` + - ``str | None`` + - optional; default ``null`` + - Method name for method actions. Example: ``"run"``. + * - ``module`` + - ``str`` + - required + - Python module path. Example: ``"tests.py"``. + * - ``steptype`` + - ``'PythonModuleStep'`` + - required + - Canonical registered step type. Example: ``"PythonModuleStep"``. + +.. _recipe-v2-step-sshclosestep: + +SSHCloseStep +~~~~~~~~~~~~ + +Closes the SSH client stored in recipe globals. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'SSHCloseStep'`` + - required + - Canonical registered step type. Example: ``"SSHCloseStep"``. + +.. _recipe-v2-step-sshconnectstep: + +SSHConnectStep +~~~~~~~~~~~~~~ + +Opens the SSH client stored in recipe globals. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'SSHConnectStep'`` + - required + - Canonical registered step type. Example: ``"SSHConnectStep"``. + +.. _recipe-v2-step-sshuploadstep: + +SSHUploadStep +~~~~~~~~~~~~~ + +Uploads files through an SSH connection. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``files`` + - ``list[UploadFile]`` + - required + - Local and remote file pairs to upload. Example: ``[{"local": "bin/tool", "remote": "/tmp/tool"}]``. + * - ``local_package`` + - ``str | None`` + - optional; default ``null`` + - Optional package containing local resources. Example: ``"my_package"``. + * - ``permissions`` + - ``int | str | None`` + - optional; default ``null`` + - Optional remote permissions. Example: ``"0755"``. + * - ``skip_if_sha256_match`` + - ``bool`` + - optional; default ``false`` + - Skip files whose remote checksum matches. Example: ``false``. + * - ``steptype`` + - ``'SSHUploadStep'`` + - required + - Canonical registered step type. Example: ``"SSHUploadStep"``. + +.. _recipe-v2-step-sequencestep: + +SequenceStep +~~~~~~~~~~~~ + +Runs another sequence as a step. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``sequence`` + - ``InternalSequenceReference`` + - required + - Internal sequence reference. Example: ``{"name": "Calibration", "type": "internal"}``. + * - ``steptype`` + - ``'SequenceStep'`` + - required + - Canonical registered step type. Example: ``"SequenceStep"``. + +.. _recipe-v2-step-serialnumberstep: + +SerialNumberStep +~~~~~~~~~~~~~~~~ + +Captures the device serial number. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'SerialNumberStep'`` + - required + - Canonical registered step type. Example: ``"SerialNumberStep"``. + +.. _recipe-v2-step-userinteractionstep: + +UserInteractionStep +~~~~~~~~~~~~~~~~~~~ + +Displays an operator interaction prompt. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'UserInteractionStep'`` + - required + - Canonical registered step type. Example: ``"UserInteractionStep"``. + +.. _recipe-v2-step-userloadingstep: + +UserLoadingStep +~~~~~~~~~~~~~~~ + +Prompts the operator to select a file. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``file_save_location`` + - ``FileDestination | None`` + - optional; default ``null`` + - Local or global destination for the selected file. Example: ``{"type": "local", "variable": "selected_file"}``. + * - ``steptype`` + - ``'UserLoadingStep'`` + - required + - Canonical registered step type. Example: ``"UserLoadingStep"``. + +.. _recipe-v2-step-userrunmethodstep: + +UserRunMethodStep +~~~~~~~~~~~~~~~~~ + +Optionally runs a Python method after an operator response. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``action_type`` + - ``str | None`` + - optional; default ``null`` + - Optional Python action type. Example: ``"method"``. + * - ``method_name`` + - ``str | None`` + - optional; default ``null`` + - Optional Python method name. Example: ``"run"``. + * - ``module`` + - ``str | None`` + - optional; default ``null`` + - Optional Python module path. Example: ``"tests.py"``. + * - ``steptype`` + - ``'UserRunMethodStep'`` + - required + - Canonical registered step type. Example: ``"UserRunMethodStep"``. + * - ``trigger_response`` + - ``str | list[any] | object | None`` + - optional; default ``null`` + - Operator response that triggers execution. Example: ``"run"``. + +.. _recipe-v2-step-userwritestep: + +UserWriteStep +~~~~~~~~~~~~~ + +Writes an operator-provided value to a configured destination. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'UserWriteStep'`` + - required + - Canonical registered step type. Example: ``"UserWriteStep"``. + +.. _recipe-v2-step-waitstep: + +WaitStep +~~~~~~~~ + +Waits for a non-negative duration in seconds. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``steptype`` + - ``'WaitStep'`` + - required + - Canonical registered step type. Example: ``"WaitStep"``. + +Input mappings +-------------- + +.. _recipe-v2-input-direct: + +DirectInput +~~~~~~~~~~~ + +Provides a literal value. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``indexed`` + - ``bool`` + - optional; default ``false`` + - Expand a list into indexed steps. Example: ``false``. + * - ``type`` + - ``'direct'`` + - required + - Input source type. Example: ``"direct"``. + * - ``value`` + - ``any`` + - required + - Literal input value. Example: ``1``. + +.. _recipe-v2-input-global: + +GlobalInput +~~~~~~~~~~~ + +Reads a recipe-global variable. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``global_name`` + - ``str`` + - required + - Global variable name. Example: ``"global_value"``. + * - ``type`` + - ``'global'`` + - required + - Input source type. Example: ``"global"``. + +.. _recipe-v2-input-local: + +LocalInput +~~~~~~~~~~ + +Reads a sequence-local variable. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``local_name`` + - ``str`` + - required + - Local variable name. Example: ``"local_value"``. + * - ``type`` + - ``'local'`` + - required + - Input source type. Example: ``"local"``. + +.. _recipe-v2-input-method: + +MethodInput +~~~~~~~~~~~ + +Resolves a method reference for the step. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'method'`` + - required + - Input source type. Example: ``"method"``. + * - ``value`` + - ``any`` + - required + - Method reference. Example: ``"helper"``. + +Output mappings +--------------- + +.. _recipe-v2-output-equals: + +EqualsOutput +~~~~~~~~~~~~ + +Passes when the output equals the configured value. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'equals'`` + - required + - Output mapping type. Example: ``"equals"``. + * - ``value`` + - ``any`` + - required + - Expected value. Example: ``3``. + +.. _recipe-v2-output-global: + +GlobalOutput +~~~~~~~~~~~~ + +Stores the output in a recipe-global variable. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``global_name`` + - ``str`` + - required + - Global destination variable. Example: ``"saved"``. + * - ``type`` + - ``'global'`` + - required + - Output mapping type. Example: ``"global"``. + +.. _recipe-v2-output-image: + +ImageOutput +~~~~~~~~~~~ + +Publishes an image output for presentation. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'image'`` + - required + - Output mapping type. Example: ``"image"``. + +.. _recipe-v2-output-local: + +LocalOutput +~~~~~~~~~~~ + +Stores the output in a sequence-local variable. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``local_name`` + - ``str`` + - required + - Local destination variable. Example: ``"saved"``. + * - ``type`` + - ``'local'`` + - required + - Output mapping type. Example: ``"local"``. + +.. _recipe-v2-output-passfail: + +PassFailOutput +~~~~~~~~~~~~~~ + +Interprets the output as a pass/fail verdict. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'passfail'`` + - required + - Output mapping type. Example: ``"passfail"``. + +.. _recipe-v2-output-passthrough: + +PassthroughOutput +~~~~~~~~~~~~~~~~~ + +Uses the nested result without adding a verdict. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``type`` + - ``'passthrough'`` + - required + - Output mapping type. Example: ``"passthrough"``. + +.. _recipe-v2-output-range: + +RangeOutput +~~~~~~~~~~~ + +Passes when the output is within an inclusive range. + +.. list-table:: Fields + :header-rows: 1 + :widths: 18 19 25 38 + + * - Field + - Type + - Requirement + - Description and example + * - ``max`` + - ``any`` + - required + - Maximum accepted value. Example: ``4``. + * - ``min`` + - ``any`` + - required + - Minimum accepted value. Example: ``1``. + * - ``type`` + - ``'range'`` + - required + - Output mapping type. Example: ``"range"``. diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 34e18cd..149fae5 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -193,7 +193,7 @@ The pyproject file operates similarily to a makefile and is the construction of 1. Define your Recipe (`my_recipe.yaml`) ----------------------------------------- -Create a YAML file defining your test sequence. The recipe consists of a main document defining metadata and global variables, followed by documents defining named sequences. See :ref:`_yaml_format` for full explaination of all steps available for recipe. +Create a YAML file defining your test sequence. The recipe consists of a main document defining metadata and global variables, followed by documents defining named sequences. See :ref:`yaml_format` for full explaination of all steps available for recipe. .. code-block:: yaml :caption: my_recipe.yaml @@ -257,7 +257,7 @@ Create a YAML file defining your test sequence. The recipe consists of a main do Alternative: Resource-Based Module Loading -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For better distribution and deployment, you can use resource-based module loading by organizing your test modules as Python packages: @@ -464,7 +464,7 @@ Test that your modules can be imported: import my_project.tests.test_module1 Common Migration Issues -~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^ **Import Errors**: Make sure all directories have ``__init__.py`` files @@ -545,7 +545,7 @@ As the recipe executes: 6. Creating and Editing Recipes with Recipe Creator tool ---------------------------------------------- +--------------------------------------------------------- For users who prefer a visual approach to recipe creation and editing, the Recipe Creator tool provides an interactive recipe editor. In the codebase this editor diff --git a/docs/source/yaml_format.rst b/docs/source/yaml_format.rst index 78a9657..23e42fc 100644 --- a/docs/source/yaml_format.rst +++ b/docs/source/yaml_format.rst @@ -2,6 +2,15 @@ .. .. SPDX-License-Identifier: CC-BY-SA-4.0 +.. important:: + + This page documents the recipe syntax currently accepted by the production + runtime (recipe language version 1). The accepted future Pydantic design is + recipe language ``2.0.0``; see :doc:`recipe_language_architecture` for its + architecture and :doc:`recipe_language_reference` for its generated syntax + reference. Do not use the version 2 example with production execution until + the integration phase is complete. + .. _yaml_format: #################### @@ -18,20 +27,20 @@ Document 1: Main Recipe Configuration .. code-block:: yaml :caption: Example Main Recipe Document - --- - name: Name of the recipe. Typically the project name. - version: Allows for tracking different versions of the file - recipe_version: Optional version of the recipe format specification. - description: A more complete description of this recipe - main_sequence: Main # Optional: Name of the sequence to run by default. Defaults typically to "Main". - test_package: my_package.tests # Optional: Python package containing test modules for PythonModuleStep - continue_on_error: false # Optional policy overriding every step's continue_on_error value. - globals: # Globals can be referenced and used from any step in the whole file - global_name: value - other_global: other_value - # ... - # tags: # Optional tags (Currently commented out in code) - # key1: value1 + --- + name: Name of the recipe. Typically the project name. + version: Allows for tracking different versions of the file + recipe_version: Optional version of the recipe format specification. + description: A more complete description of this recipe + main_sequence: Main # Optional: Name of the sequence to run by default. Defaults typically to "Main". + test_package: my_package.tests # Optional: Python package containing test modules for PythonModuleStep + continue_on_error: false # Optional policy overriding every step's continue_on_error value. + globals: # Globals can be referenced and used from any step in the whole file + global_name: value + other_global: other_value + # ... + # tags: # Optional tags (Currently commented out in code) + # key1: value1 Main Recipe Configuration Fields ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -392,23 +401,33 @@ available for later steps or reporting: **Summary of assertion vs. storage types:** -+---------------+---------------------------+----------------------------+ -| Type | Behaviour | Determines Pass/Fail? | -+===============+===========================+============================+ -| ``passfail`` | Boolean → PASS / FAIL | Yes | -+---------------+---------------------------+----------------------------+ -| ``equals`` | Compare to target value | Yes | -+---------------+---------------------------+----------------------------+ -| ``range`` | Check within [min, max] | Yes | -+---------------+---------------------------+----------------------------+ -| ``passthrough``| Propagate a ResultType | Yes | -+---------------+---------------------------+----------------------------+ -| ``global`` | Store in global variable | No — step returns DONE | -+---------------+---------------------------+----------------------------+ -| ``local`` | Store in local variable | No — step returns DONE | -+---------------+---------------------------+----------------------------+ -| ``image`` | Copy image file into report and embed in HTML | No — step returns DONE | -+---------------+---------------------------+----------------------------+ +.. list-table:: + :header-rows: 1 + + * - Type + - Behaviour + - Determines pass/fail? + * - ``passfail`` + - Boolean → PASS / FAIL + - Yes + * - ``equals`` + - Compare to target value + - Yes + * - ``range`` + - Check within [min, max] + - Yes + * - ``passthrough`` + - Propagate a ResultType + - Yes + * - ``global`` + - Store in global variable + - No — step returns DONE + * - ``local`` + - Store in local variable + - No — step returns DONE + * - ``image`` + - Copy image file into report and embed in HTML + - No — step returns DONE .. note:: If a step's output mapping contains **only** ``global`` and/or ``local`` @@ -483,7 +502,7 @@ Each has its own template of required elements but with overlapping types of ele The elements ``step_name`` and ``description`` are not explained further in this section as they're descriptive elements for reporting and GUI with no change between steps. .. note:: - The optional arguments ``critical``, ``skip`` and ``continue_on_error`` all apply to the following step types. Check :ref:`_step_definition_details` for information on ``skip`` and ``critical`` and :ref:`_continue_on_error_details` for information on ``continue_on_error``. + The optional arguments ``critical``, ``skip`` and ``continue_on_error`` all apply to the following step types. Check :ref:`step_definition_details` for information on ``skip`` and ``critical`` and :ref:`continue_on_error_details` for information on ``continue_on_error``. @@ -558,9 +577,10 @@ Allows for user to interact with gui through action on buttons. It allows for ad * ``steptype`` (str): Determines the type of action. * ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. * ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to `True`. +* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. + These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to `True` and `False`. +The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. * ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. * ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. @@ -581,13 +601,14 @@ Allows to setup a SSH connection to be used globally during the test. * ``steptype`` (str): Determines the type of action. -This steptype has certain **requirements** to which globals exist. The following required are +This steptype has certain **requirements** for which globals exist. The required globals are: + * ``ssh_client: None`` : The variable holding the opened paramiko client to be called in functions. * ``host: 129`` :The SSH hostname or IP address. * ``user: username``:The SSH username * ``password: None`` :Password for SSH auth. * ``private_key: 'path/to/your/key_file'`` :The path to key file. important if no password is given. -* ``port: None``(int) : SSH port (default: 22). +* ``port: None`` (int): SSH port (default: 22). If password is not supplied, the function will automatically use ``private_key`` as verification. @@ -610,6 +631,7 @@ To use the SSH client, add ``ssh_client`` to input_mapping. An Example of using it for a function can be seen below. .. code-block:: python + def write_a_simple_filessh(target): target.exec_command("echo 'Hello World' > myfile.txt") @@ -695,9 +717,10 @@ Used to load a file to be used somewhere else. Could fx be a calibration file or * ``steptype`` (str): Determines the type of action. * ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. * ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to `True`. +* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. + These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to `True` and `False`. +The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. * ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. * ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. @@ -750,9 +773,10 @@ Executes a method after interacting with next button. Step type is expected to b * ``module`` (str): Python module path. With ``test_package``, it is relative to that package and may include nested directories. * ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. * ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to `True`. +* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. + These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to `True` and `False`. +The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. * ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. * ``method_name`` (dict): specifies the method to run. * ``argument1-3`` (dict): Any dict that is not message, option or image path in input mapping will be considered input to method. multiple inputs are possible with them being ordered from top to bottom as inputs to the method. @@ -762,6 +786,7 @@ The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop Example function of input would be for the step above. .. code-block:: python + def simpleMethod(argument1, argument2, argument3) #the input sequence seen from the above step. It shows the order of the return @@ -798,8 +823,9 @@ Executes step to write values to variables or setting up the settings required f * ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. * ``message`` (dict): can write a message that is expected to be relevant for user to do before * ``options`` (dict): options to add buttons. For the main functionality of this function, two keys are defined: ``'ID'`` for setting up a comport through gui and ``'wrt'`` for writing a string to a variable. + These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to `True` and `False`. +The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. * ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. * ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. @@ -807,14 +833,16 @@ The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop This step requires some local variables depending on which **key** is specified under ``options``. If the **key** chosen is ``'ID'``, it requires the following local variables. + * serial_ID * serialport * baudrate + When the key is applied and button is pushed, a GUI pops up letting you choose baudrate and what comport that is available you want to connect to. you can send an IDN? command through button and when found to work it will save the values to the local variables. The output mapping should just be pass/faill for this key. If the **key** chosen is ``'wrt'``, it requires an output mapping of either global or local scale to a variable. -The input written into the GUI window is sent to the output mapping to be saved as a ``str``. The following is the required ``output_mapping`` if the **key** is ``'wrt'` on a button. +The input written into the GUI window is sent to the output mapping to be saved as a ``str``. The following is the required ``output_mapping`` if the **key** is ``'wrt'`` on a button. .. code-block:: yaml @@ -897,38 +925,29 @@ in the main sequence, but it can equally be placed inside a setup or sub-sequenc Required globals and locals for certain steps. -============================ +================================================= To ensure the functionality of some of the step types, certain global and locals are required for different datatypes. This section explains which step requires what specific variables in the recipe. The following steps that require global or local variables are found below. - - **SSHConnectStep** - - Requires the following global variables: - - cancel_key: 'cancel' - - ssh_client: None - - host: Ip of the host - - user: root or user - - password: None - - private_key: 'path/to/private_key' - - port: SSH port. standard is 22 - - **UserLoadingStep** - Requires the following global variables: - - cancel_key: 'cancel' - - loadFile_key: 'file' - - **UserRunMethodStep** -Requires the following global variables: - - cancel_key: 'cancel' -- **UserWriteStep** -Requires the following global variables: - - cancel_key: 'cancel' - - ID_key: 'ID' - - wrt_key: 'wrt' - -Requires the following local variables **only** if ID_key is specified under options: - - serial_ID: None - - serialport: None - - baudrate: None +* **SSHConnectStep** + + Requires the ``cancel_key``, ``ssh_client``, ``host``, ``user``, ``port``, + and either ``password`` or ``private_key`` globals. + +* **UserLoadingStep** + + Requires the ``cancel_key`` and ``loadFile_key`` globals. + +* **UserRunMethodStep** + + Requires the ``cancel_key`` global. + +* **UserWriteStep** + + Requires the ``cancel_key``, ``ID_key``, and ``wrt_key`` globals. If + ``ID_key`` is specified in the options, it also requires the ``serial_ID``, + ``serialport``, and ``baudrate`` local variables. - **SerialNumberStep** diff --git a/reuse.toml b/reuse.toml index 8608766..12310a8 100644 --- a/reuse.toml +++ b/reuse.toml @@ -41,6 +41,9 @@ path = [ "docs/**/*.rst", "docs/**/*.md", "docs/**/*.txt", + "docs/**/*.json", + "docs/**/*.yml", + "docs/**/*.yaml", "docs/INSTALL" ] precedence = "aggregate" diff --git a/spikes/recipe_pydantic/__init__.py b/spikes/recipe_pydantic/__init__.py index dfbf6e5..a9eb705 100644 --- a/spikes/recipe_pydantic/__init__.py +++ b/spikes/recipe_pydantic/__init__.py @@ -1,17 +1,36 @@ """Isolated Pydantic prototype for recipe-language version 2.""" -from .models import Recipe -from .parser import ( - Diagnostic, - ParseResult, - RecipeParseError, - SourcePosition, - SourceSpan, - dump_recipe, - parse_recipe_file, - parse_recipe_text, -) -from .reference import render_json_schema, render_reference +from importlib import import_module +from typing import Any + +_PARSER_EXPORTS = { + "Diagnostic", + "ParseResult", + "RecipeParseError", + "SourcePosition", + "SourceSpan", + "dump_recipe", + "parse_recipe_file", + "parse_recipe_text", +} + + +def __getattr__(name: str) -> Any: + """Keep JSON-only submodules importable without initializing Pydantic.""" + if name == "Recipe": + return getattr(import_module(".models", __name__), name) + if name in _PARSER_EXPORTS: + return getattr(import_module(".parser", __name__), name) + raise AttributeError(name) + + +def render_json_schema() -> str: + return import_module(".artifacts", __name__).render_json_schema() + + +def render_reference() -> str: + return import_module(".artifacts", __name__).rendered_artifacts()[1] + __all__ = [ "Diagnostic", diff --git a/spikes/recipe_pydantic/artifacts.py b/spikes/recipe_pydantic/artifacts.py new file mode 100644 index 0000000..e65358b --- /dev/null +++ b/spikes/recipe_pydantic/artifacts.py @@ -0,0 +1,89 @@ +"""Generate or check the documentation artifacts for recipe language 2.0.""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Sequence +from pathlib import Path + +from .models import Recipe +from .reference import render_reference + +ROOT = Path(__file__).parents[2] +DEFAULT_SCHEMA_PATH = ROOT / "docs" / "source" / "_static" / "recipe_language.schema.json" +DEFAULT_REFERENCE_PATH = ROOT / "docs" / "source" / "recipe_language_reference.rst" + + +def render_json_schema() -> str: + """Render deterministic JSON Schema from the accepted Pydantic model.""" + schema = Recipe.model_json_schema(by_alias=True, mode="validation") + schema["$comment"] = ( + "SPDX-FileCopyrightText: 2026 CERN ; " + "SPDX-License-Identifier: CC-BY-SA-4.0" + ) + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def rendered_artifacts() -> tuple[str, str]: + schema_text = render_json_schema() + reference_text = render_reference(json.loads(schema_text)) + return schema_text, reference_text + + +def write_artifacts( + schema_path: str | Path = DEFAULT_SCHEMA_PATH, + reference_path: str | Path = DEFAULT_REFERENCE_PATH, +) -> None: + schema_text, reference_text = rendered_artifacts() + for path, content in ( + (Path(schema_path), schema_text), + (Path(reference_path), reference_text), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def check_artifacts( + schema_path: str | Path = DEFAULT_SCHEMA_PATH, + reference_path: str | Path = DEFAULT_REFERENCE_PATH, +) -> bool: + expected = rendered_artifacts() + try: + current = ( + Path(schema_path).read_text(encoding="utf-8"), + Path(reference_path).read_text(encoding="utf-8"), + ) + except (OSError, UnicodeError): + return False + return current == expected + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if either artifact is stale") + parser.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA_PATH) + parser.add_argument("--reference", type=Path, default=DEFAULT_REFERENCE_PATH) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + if arguments.check: + if check_artifacts(arguments.schema, arguments.reference): + return 0 + print("Recipe documentation artifacts are missing or stale.", file=sys.stderr) + return 1 + try: + write_artifacts(arguments.schema, arguments.reference) + except OSError as error: + print(f"Could not write recipe documentation artifacts: {error}", file=sys.stderr) + return 2 + print(f"Wrote {arguments.schema}") + print(f"Wrote {arguments.reference}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spikes/recipe_pydantic/models.py b/spikes/recipe_pydantic/models.py index e616392..93a5f59 100644 --- a/spikes/recipe_pydantic/models.py +++ b/spikes/recipe_pydantic/models.py @@ -40,6 +40,7 @@ class DirectInput(RecipeModel): exclude_if=lambda value: not value, ) + # docs:indexed-direct-start @model_validator(mode="after") def indexed_values_are_lists(self) -> DirectInput: if self.indexed and not isinstance(self.value, list): @@ -47,6 +48,7 @@ def indexed_values_are_lists(self) -> DirectInput: "invalid_indexed_input", "Indexed direct input value must be a list." ) return self + # docs:indexed-direct-end class LocalInput(RecipeModel): @@ -187,6 +189,7 @@ class PythonModuleStep(CommonStep): module: str = described("Python module path.", example="tests.py") method_name: str | None = described("Method name for method actions.", example="run", default=None) + # docs:method-name-start @model_validator(mode="after") def method_actions_have_names(self) -> PythonModuleStep: if self.action_type == "method" and not self.method_name: @@ -194,6 +197,7 @@ def method_actions_have_names(self) -> PythonModuleStep: "missing_method_name", "Method actions require method_name." ) return self + # docs:method-name-end class SequenceStep(CommonStep): @@ -220,11 +224,13 @@ class WaitStep(CommonStep): steptype: Literal["WaitStep"] = described("Canonical registered step type.", example="WaitStep") + # docs:wait-time-start @model_validator(mode="after") def has_wait_time(self) -> WaitStep: if "wait_time" not in self.input_mapping: raise PydanticCustomError("missing_required_input", "WaitStep requires input 'wait_time'.") return self + # docs:wait-time-end class UserLoadingStep(CommonStep): diff --git a/spikes/recipe_pydantic/parser.py b/spikes/recipe_pydantic/parser.py index 83bf4cc..729dd78 100644 --- a/spikes/recipe_pydantic/parser.py +++ b/spikes/recipe_pydantic/parser.py @@ -305,6 +305,7 @@ def _semantic_diagnostics( ) -> list[Diagnostic]: """Rules that cannot be expressed by one structural Pydantic model.""" diagnostics: list[Diagnostic] = [] + # docs:sequence-semantics-start by_name: dict[str, tuple[int, Sequence]] = {} for document_index, sequence in sequences: path = (document_index, "sequence_name") @@ -327,12 +328,14 @@ def _semantic_diagnostics( source_name, spans, )) + # docs:sequence-semantics-end verdict_types = (PassFailOutput, EqualsOutput, RangeOutput, PassthroughOutput) for document_index, sequence in sequences: flattened = list(_all_steps(sequence)) for section, index, step in flattened: step_path = (document_index, section, index) + # docs:nested-reference-start if isinstance(step, SequenceStep) and step.sequence.name not in by_name: diagnostics.append(_diagnostic( "unknown-sequence-reference", @@ -342,7 +345,9 @@ def _semantic_diagnostics( source_name, spans, )) + # docs:nested-reference-end + # docs:mapping-semantics-start indexed_lengths = [ len(value.value) for value in step.input_mapping.values() @@ -368,7 +373,9 @@ def _semantic_diagnostics( source_name, spans, )) + # docs:mapping-semantics-end + # docs:ssh-semantics-start ssh_steps = [item for item in flattened if item[2].steptype.startswith("SSH")] if ssh_steps and header is not None: for required in ("ssh_client", "host", "user", "port"): @@ -414,6 +421,7 @@ def _semantic_diagnostics( source_name, spans, )) + # docs:ssh-semantics-end return diagnostics diff --git a/spikes/recipe_pydantic/reference.py b/spikes/recipe_pydantic/reference.py index a66bbb0..1879c6b 100644 --- a/spikes/recipe_pydantic/reference.py +++ b/spikes/recipe_pydantic/reference.py @@ -1,128 +1,213 @@ -"""JSON Schema and compact RST renderers driven only by Pydantic metadata.""" +"""Render the Sphinx recipe reference from committed JSON Schema only.""" from __future__ import annotations import json -import re from pathlib import Path from typing import Any -from pydantic import BaseModel -from pydantic_core import PydanticUndefined - -from .models import ( - INPUT_MODELS, - OUTPUT_MODELS, - STEP_MODELS, - FileDestination, - InternalSequenceReference, - Recipe, - RecipeHeader, - Sequence, - UploadFile, -) - -ROOT = Path(__file__).parent -SCHEMA_PATH = ROOT / "recipe.schema.json" -REFERENCE_PATH = ROOT / "recipe_reference.rst" - - -def render_json_schema() -> str: - """Render deterministic JSON Schema for the aggregate typed model.""" - schema = Recipe.model_json_schema(by_alias=True, mode="validation") - schema["$comment"] = ( - "SPDX-FileCopyrightText: 2026 CERN ; " - "SPDX-License-Identifier: LGPL-2.1-or-later" - ) - return json.dumps( - schema, - indent=2, - sort_keys=True, - ) + "\n" - - -def _annotation_name(annotation: Any) -> str: - text = str(annotation).replace("typing.", "") - text = text.replace("", "") - text = re.sub(r"(?:[\w.]*recipe_pydantic\.models|__main__)\.", "", text) - return text.replace("NoneType", "None") - - -def _example(field: Any) -> Any: - if field.examples: - return field.examples[0] - if field.default is not PydanticUndefined: - return field.default - return PydanticUndefined - - -def _model_block(model: type[BaseModel], anchor: str) -> list[str]: - title = model.__name__ - lines = [f".. _{anchor}:", "", title, "~" * len(title), ""] - description = (model.__doc__ or "").strip() - if description: - lines.extend([description, ""]) - lines.extend([".. list-table:: Fields", " :header-rows: 1", "", " * - Field", " - Type", " - Required/default", " - Description / example"]) - for name, field in model.model_fields.items(): - public_name = field.alias or name - default = "required" if field.is_required() else f"default ``{field.default!r}``" - example = _example(field) - detail = field.description or "" - if example is not PydanticUndefined: - detail += f" Example: ``{example!r}``." - lines.extend([ - f" * - ``{public_name}``", - f" - ``{_annotation_name(field.annotation)}``", - f" - {default}", - f" - {detail}", - ]) + +def load_schema(path: str | Path) -> dict[str, Any]: + """Load and minimally verify an aggregate recipe JSON Schema.""" + source = Path(path) + schema = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(schema, dict) or not isinstance(schema.get("$defs"), dict): + raise TypeError(f"Recipe schema has no $defs object: {source}") + return schema + + +def _reference_name(reference: str) -> str: + prefix = "#/$defs/" + if not reference.startswith(prefix): + raise ValueError(f"Unsupported external JSON Schema reference: {reference}") + return reference.removeprefix(prefix) + + +def _resolve(value: dict[str, Any], definitions: dict[str, Any]) -> dict[str, Any]: + if "$ref" not in value: + return value + return definitions[_reference_name(value["$ref"])] + + +def _type_name(value: dict[str, Any]) -> str: + if "$ref" in value: + return _reference_name(value["$ref"]) + if "const" in value: + return repr(value["const"]) + if "enum" in value: + return " | ".join(repr(item) for item in value["enum"]) + if "anyOf" in value: + return " | ".join(_type_name(item) for item in value["anyOf"]) + kind = value.get("type") + if kind == "array": + return f"list[{_type_name(value.get('items', {}))}]" + if kind == "object": + additional = value.get("additionalProperties") + if isinstance(additional, dict): + return f"dict[str, {_type_name(additional)}]" + return "object" + return { + "boolean": "bool", + "integer": "int", + "number": "number", + "null": "None", + "string": "str", + }.get(kind, "any") + + +def _literal(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def _field_table( + definition: dict[str, Any], + *, + include: set[str] | None = None, + exclude: set[str] | None = None, +) -> list[str]: + properties = definition.get("properties", {}) + required = set(definition.get("required", [])) + names = [ + name for name in properties + if (include is None or name in include) and (exclude is None or name not in exclude) + ] + if not names: + return ["This variant adds no fields.", ""] + # REUSE-IgnoreStart + lines = [ + ".. list-table:: Fields", + " :header-rows: 1", + " :widths: 18 19 25 38", + "", + " * - Field", + " - Type", + " - Requirement", + " - Description and example", + ] + for name in names: + field = properties[name] + requirement = "required" if name in required else "optional" + if "default" in field: + requirement += f"; default ``{_literal(field['default'])}``" + details = field.get("description", "") + examples = field.get("examples", []) + if examples: + details += f" Example: ``{_literal(examples[0])}``." + lines.extend(( + f" * - ``{name}``", + f" - ``{_type_name(field)}``", + f" - {requirement}", + f" - {details}", + )) lines.append("") return lines -def render_reference() -> str: - """Render the review-oriented recipe reference from model metadata.""" +def _model_section( + definition_name: str, + definitions: dict[str, Any], + anchor: str, + *, + include: set[str] | None = None, + exclude: set[str] | None = None, +) -> list[str]: + definition = definitions[definition_name] + title = definition.get("title", definition_name) + lines = [f".. _{anchor}:", "", title, "~" * len(title), ""] + if definition.get("description"): + lines.extend((definition["description"], "")) + lines.extend(_field_table(definition, include=include, exclude=exclude)) + return lines + + +def _discriminator_mapping( + definitions: dict[str, Any], name: str +) -> dict[str, str]: + definition = definitions[name] + mapping = definition.get("discriminator", {}).get("mapping") + if not isinstance(mapping, dict) or not mapping: + raise ValueError(f"$defs.{name} has no discriminator mapping") + return {key: _reference_name(reference) for key, reference in mapping.items()} + + +def _common_step_fields( + definitions: dict[str, Any], step_names: list[str] +) -> set[str]: + property_sets = [set(definitions[name].get("properties", {})) for name in step_names] + common = set.intersection(*property_sets) + result: set[str] = set() + for field_name in common: + values = [definitions[name]["properties"][field_name] for name in step_names] + if all(value == values[0] for value in values[1:]): + result.add(field_name) + return result + + +def render_reference(schema: dict[str, Any]) -> str: + """Render deterministic RST using only a parsed JSON Schema document.""" + definitions = schema["$defs"] + steps = _discriminator_mapping(definitions, "Step") + inputs = _discriminator_mapping(definitions, "InputMapping") + outputs = _discriminator_mapping(definitions, "OutputMapping") + common_fields = _common_step_fields(definitions, list(steps.values())) + lines = [ ".. SPDX-FileCopyrightText: 2026 CERN ", - ".. SPDX-License-Identifier: LGPL-2.1-or-later", - ".. This file is generated by the Pydantic recipe spike.", + "..", + ".. SPDX-License-Identifier: CC-BY-SA-4.0", + "..", + ".. Generated from recipe_language.schema.json. Do not edit manually.", + "", + "Recipe Language 2.0 Reference", + "=============================", "", - "Pydantic Recipe Language 2.0.0", - "================================", + "This page is generated from the tracked aggregate JSON Schema. It describes", + "the accepted future recipe language model; production execution still uses", + "the version 1 language until Phase 6 integration is complete.", "", - "This candidate reference is generated directly from strict, frozen Pydantic models.", + ":download:`Download the JSON Schema <_static/recipe_language.schema.json>`.", + "", + "See :doc:`recipe_language_architecture` for parsing, semantic rules,", + "documentation maintenance, and the planned YamVIEW and sequencer flows.", "", "Documents", "---------", "", ] - lines.extend(_model_block(RecipeHeader, "recipe-header")) - lines.extend(_model_block(Sequence, "recipe-sequence")) - - lines.extend(["Nested structures", "-----------------", ""]) - for model in (InternalSequenceReference, FileDestination, UploadFile): - lines.extend(_model_block(model, f"recipe-structure-{model.__name__.lower()}")) - - lines.extend(["Authorable steps", "----------------", ""]) - for model in STEP_MODELS: - lines.extend(_model_block(model, f"recipe-step-{model.__name__.lower()}")) - - lines.extend(["Input mappings", "--------------", ""]) - for model in INPUT_MODELS: - kind = model.model_fields["type"].examples[0] - lines.extend(_model_block(model, f"recipe-input-{kind}")) - - lines.extend(["Output mappings", "---------------", ""]) - for model in OUTPUT_MODELS: - kind = model.model_fields["type"].examples[0] - lines.extend(_model_block(model, f"recipe-output-{kind}")) + # REUSE-IgnoreEnd + lines.extend(_model_section("RecipeHeader", definitions, "recipe-v2-header")) + lines.extend(_model_section("Sequence", definitions, "recipe-v2-sequence")) + + lines.extend(("Nested structures", "-----------------", "")) + for name in ("InternalSequenceReference", "FileDestination", "UploadFile"): + lines.extend(_model_section(name, definitions, f"recipe-v2-structure-{name.lower()}")) + + lines.extend(("Common step fields", "------------------", "")) + representative = next(iter(steps.values())) + lines.extend(_field_table(definitions[representative], include=common_fields)) + + lines.extend(("Authorable steps", "----------------", "")) + for discriminator, definition_name in steps.items(): + lines.extend(_model_section( + definition_name, + definitions, + f"recipe-v2-step-{discriminator.lower()}", + exclude=common_fields, + )) + + lines.extend(("Input mappings", "--------------", "")) + for discriminator, definition_name in inputs.items(): + lines.extend(_model_section( + definition_name, definitions, f"recipe-v2-input-{discriminator}" + )) + + lines.extend(("Output mappings", "---------------", "")) + for discriminator, definition_name in outputs.items(): + lines.extend(_model_section( + definition_name, definitions, f"recipe-v2-output-{discriminator}" + )) return "\n".join(lines).rstrip() + "\n" -def write_artifacts() -> None: - SCHEMA_PATH.write_text(render_json_schema(), encoding="utf-8") - REFERENCE_PATH.write_text(render_reference(), encoding="utf-8") - - -if __name__ == "__main__": - write_artifacts() +def render_reference_file(path: str | Path) -> str: + return render_reference(load_schema(path)) diff --git a/spikes/recipe_pydantic/test_recipe_pydantic.py b/spikes/recipe_pydantic/test_recipe_pydantic.py index a0d62d3..ff4058d 100644 --- a/spikes/recipe_pydantic/test_recipe_pydantic.py +++ b/spikes/recipe_pydantic/test_recipe_pydantic.py @@ -15,9 +15,14 @@ from pypts.recipe_parser import dump_recipe as dump_v1 from pypts.recipe_parser import parse_recipe_file as parse_v1 +from .artifacts import ( + DEFAULT_REFERENCE_PATH, + DEFAULT_SCHEMA_PATH, + render_json_schema, +) from .models import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS from .parser import RecipeParseError, dump_recipe, parse_recipe_file, parse_recipe_text -from .reference import REFERENCE_PATH, SCHEMA_PATH, render_json_schema, render_reference +from .reference import render_reference ROOT = Path(__file__).parents[2] RECIPES = ROOT / "src" / "pypts" / "recipes" @@ -368,21 +373,22 @@ def test_raw_legacy_corpus_exposes_migration_diagnostics(): def test_generated_schema_and_reference_are_complete_and_current(): schema_text = render_json_schema() - reference = render_reference() - assert schema_text == SCHEMA_PATH.read_text(encoding="utf-8") - assert reference == REFERENCE_PATH.read_text(encoding="utf-8") - definitions = json.loads(schema_text)["$defs"] + schema = json.loads(schema_text) + reference = render_reference(schema) + assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") + assert reference == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + definitions = schema["$defs"] for model in STEP_MODELS + INPUT_MODELS + OUTPUT_MODELS: assert model.__name__ in definitions kind = model.model_fields.get("steptype") or model.model_fields["type"] anchor_kind = kind.examples[0].lower() group = "step" if model in STEP_MODELS else "input" if model in INPUT_MODELS else "output" - assert reference.count(f".. _recipe-{group}-{anchor_kind}:") == 1 + assert reference.count(f".. _recipe-v2-{group}-{anchor_kind}:") == 1 report = STEP_MODELS[0].model_fields["skip"] assert report.description in reference - assert f"default ``{report.default!r}``" in reference - assert f"Example: ``{report.examples[0]!r}``." in reference + assert 'default ``false``' in reference + assert 'Example: ``false``.' in reference def test_spike_has_no_runtime_gui_yamview_or_sphinx_imports(): diff --git a/src/pypts/recipe_reference.py b/src/pypts/recipe_reference.py index 21e28e8..c966fa1 100644 --- a/src/pypts/recipe_reference.py +++ b/src/pypts/recipe_reference.py @@ -6,9 +6,9 @@ from __future__ import annotations import argparse +import sys from collections.abc import Mapping, Sequence from pathlib import Path -import sys from typing import Any import yaml @@ -17,18 +17,17 @@ CANONICAL_RECIPE_VERSION, COMMON_STEP_FIELDS, CONSTRAINT_SPECS, - FieldSpec, HEADER_SPEC, INPUT_MAPPING_SPECS, - MappingSpec, OUTPUT_MAPPING_SPECS, SEQUENCE_SPEC, STEP_SPECS, + FieldSpec, + MappingSpec, StepSpec, ) from pypts.recipe_parser import dump_recipe, parse_recipe_text - DEFAULT_REFERENCE_PATH = Path("docs/generated/recipe_language_reference.rst") @@ -154,6 +153,7 @@ def _requirement(field: FieldSpec) -> str: def _field_table(fields: Sequence[FieldSpec]) -> list[str]: + # REUSE-IgnoreStart lines = [ ".. list-table::", " :header-rows: 1", @@ -233,6 +233,7 @@ def render_recipe_reference() -> str: HEADER_SPEC.description, "", ] + # REUSE-IgnoreEnd lines.extend(_field_table(HEADER_SPEC.fields)) lines.extend(("", "Sequence", "--------", "", SEQUENCE_SPEC.description, "")) lines.extend(_field_table(SEQUENCE_SPEC.fields)) diff --git a/tests/unit_tests/test_recipe_pydantic_docs.py b/tests/unit_tests/test_recipe_pydantic_docs.py new file mode 100644 index 0000000..9e138aa --- /dev/null +++ b/tests/unit_tests/test_recipe_pydantic_docs.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Phase 5 checks for schema-driven recipe-language documentation.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +from spikes.recipe_pydantic.artifacts import ( + DEFAULT_REFERENCE_PATH, + DEFAULT_SCHEMA_PATH, + check_artifacts, + main, + render_json_schema, + write_artifacts, +) +from spikes.recipe_pydantic.parser import ( + dump_recipe, + parse_recipe_file, + parse_recipe_text, +) +from spikes.recipe_pydantic.reference import render_reference + +ROOT = Path(__file__).parents[2] +DOC_RECIPE = ROOT / "docs" / "source" / "_examples" / "recipe_v2.yml" +ARCHITECTURE = ROOT / "docs" / "source" / "recipe_language_architecture.rst" +REFERENCE_RENDERER = ROOT / "spikes" / "recipe_pydantic" / "reference.py" + + +def _mapping(schema, name): + return schema["$defs"][name]["discriminator"]["mapping"] + + +def test_committed_schema_and_reference_are_current(): + schema_text = render_json_schema() + schema = json.loads(schema_text) + assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") + assert render_reference(schema) == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + assert check_artifacts() + assert main(["--check"]) == 0 + + +def test_check_mode_detects_stale_artifacts_without_writing(tmp_path): + schema_path = tmp_path / "recipe.schema.json" + reference_path = tmp_path / "reference.rst" + write_artifacts(schema_path, reference_path) + assert main([ + "--check", "--schema", str(schema_path), "--reference", str(reference_path) + ]) == 0 + + reference_path.write_text("stale\n", encoding="utf-8") + assert main([ + "--check", "--schema", str(schema_path), "--reference", str(reference_path) + ]) == 1 + assert reference_path.read_text(encoding="utf-8") == "stale\n" + + +def test_every_discriminator_is_rendered_once(): + schema = json.loads(DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8")) + reference = DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + for group, definition in ( + ("step", "Step"), + ("input", "InputMapping"), + ("output", "OutputMapping"), + ): + for discriminator in _mapping(schema, definition): + anchor = f".. _recipe-v2-{group}-{discriminator.lower()}:" + assert reference.count(anchor) == 1 + + +def test_reference_metadata_comes_from_json_schema(): + schema = json.loads(DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8")) + reference = DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + direct = schema["$defs"]["DirectInput"]["properties"]["indexed"] + assert direct["description"] in reference + assert f"default ``{json.dumps(direct['default'])}``" in reference + assert f"Example: ``{json.dumps(direct['examples'][0])}``." in reference + assert "``bool``" in reference + assert "``min``" in reference and "``max``" in reference + assert "``value``" in reference and "required" in reference + + +def test_json_only_renderer_has_no_model_runtime_ui_or_sphinx_imports(): + tree = ast.parse(REFERENCE_RENDERER.read_text(encoding="utf-8")) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + forbidden = { + "pydantic", + "spikes.recipe_pydantic.models", + "pypts.recipe", + "pypts.steps", + "pypts.YamVIEW", + "sphinx", + } + assert not any( + name == item or name.startswith(item + ".") + for name in imported + for item in forbidden + ) + + +def test_documentation_recipe_is_warning_free_and_model_stable(): + first = parse_recipe_file(DOC_RECIPE) + assert first.is_valid, first.errors + assert not first.warnings + second = parse_recipe_text(dump_recipe(first.require_recipe()), "canonical:recipe_v2.yml") + assert second.is_valid, second.errors + assert not second.warnings + assert second.recipe == first.recipe + + +def test_literalinclude_markers_are_unique_and_paired(): + architecture = ARCHITECTURE.read_text(encoding="utf-8") + sources = { + "models.py": (ROOT / "spikes" / "recipe_pydantic" / "models.py").read_text( + encoding="utf-8" + ), + "parser.py": (ROOT / "spikes" / "recipe_pydantic" / "parser.py").read_text( + encoding="utf-8" + ), + } + markers = ( + ("models.py", "indexed-direct"), + ("models.py", "method-name"), + ("models.py", "wait-time"), + ("parser.py", "sequence-semantics"), + ("parser.py", "nested-reference"), + ("parser.py", "mapping-semantics"), + ("parser.py", "ssh-semantics"), + ) + for filename, name in markers: + start = f"# docs:{name}-start" + end = f"# docs:{name}-end" + assert sources[filename].count(start) == 1 + assert sources[filename].count(end) == 1 + assert architecture.count(start) == 1 + assert architecture.count(end) == 1 + + +def test_sphinx_sources_link_reference_schema_and_example(): + index = (ROOT / "docs" / "source" / "index.rst").read_text(encoding="utf-8") + architecture = ARCHITECTURE.read_text(encoding="utf-8") + assert "recipe_language_reference" in index + assert "_static/recipe_language.schema.json" in architecture + assert "_examples/recipe_v2.yml" in architecture From 96c499c23e1bb155ba64c06c4eb9533cc052e715 Mon Sep 17 00:00:00 2001 From: alvaro Date: Thu, 13 Aug 2026 16:36:42 +0200 Subject: [PATCH 07/14] docs update on the new model --- .github/workflows/ci.yml | 5 + Dockerfile | 4 +- docs/.gitignore | 1 + .../_static/recipe_language.schema.json | 2085 ----------------- docs/source/conf.py | 28 +- docs/source/dependency_license_analysis.rst | 27 +- docs/source/index.rst | 2 +- docs/source/recipe_language_architecture.rst | 168 +- docs/source/recipe_language_reference.rst | 784 ------- docs/source/yaml_format.rst | 6 +- pyproject.toml | 4 +- spikes/recipe_pydantic/artifacts.py | 20 +- spikes/recipe_pydantic/reference.py | 9 +- tests/unit_tests/test_recipe_pydantic_docs.py | 27 +- 14 files changed, 189 insertions(+), 2981 deletions(-) delete mode 100644 docs/source/_static/recipe_language.schema.json delete mode 100644 docs/source/recipe_language_reference.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaa0932..48f1133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,8 @@ jobs: run: >- docker run --rm pts-framework-ci:${GITHUB_SHA} python -m pytest tests -vv --durations=20 + + - name: Build documentation + run: >- + docker run --rm pts-framework-ci:${GITHUB_SHA} + python -m sphinx -W --keep-going -b html docs/source docs/build/html diff --git a/Dockerfile b/Dockerfile index 5c639a7..b5221ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,10 @@ WORKDIR /app COPY pyproject.toml README.md ./ COPY src ./src COPY tests ./tests +COPY docs ./docs +COPY spikes/recipe_pydantic ./spikes/recipe_pydantic RUN python -m pip install --no-cache-dir --upgrade pip \ - && python -m pip install --no-cache-dir ".[test]" build + && python -m pip install --no-cache-dir ".[test,doc]" build CMD ["python", "-m", "pytest", "tests"] diff --git a/docs/.gitignore b/docs/.gitignore index 87b4b55..177c2b2 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -6,5 +6,6 @@ build source/api/* +source/_generated/ # End of .gitignore content managed by acc-py diff --git a/docs/source/_static/recipe_language.schema.json b/docs/source/_static/recipe_language.schema.json deleted file mode 100644 index 24382b7..0000000 --- a/docs/source/_static/recipe_language.schema.json +++ /dev/null @@ -1,2085 +0,0 @@ -{ - "$comment": "SPDX-FileCopyrightText: 2026 CERN ; SPDX-License-Identifier: CC-BY-SA-4.0", - "$defs": { - "DirectInput": { - "additionalProperties": false, - "description": "Provides a literal value.", - "properties": { - "indexed": { - "default": false, - "description": "Expand a list into indexed steps.", - "examples": [ - false - ], - "title": "Indexed", - "type": "boolean" - }, - "type": { - "const": "direct", - "description": "Input source type.", - "examples": [ - "direct" - ], - "title": "Type", - "type": "string" - }, - "value": { - "description": "Literal input value.", - "examples": [ - 1 - ], - "title": "Value" - } - }, - "required": [ - "type", - "value" - ], - "title": "DirectInput", - "type": "object" - }, - "EqualsOutput": { - "additionalProperties": false, - "description": "Passes when the output equals the configured value.", - "properties": { - "type": { - "const": "equals", - "description": "Output mapping type.", - "examples": [ - "equals" - ], - "title": "Type", - "type": "string" - }, - "value": { - "description": "Expected value.", - "examples": [ - 3 - ], - "title": "Value" - } - }, - "required": [ - "type", - "value" - ], - "title": "EqualsOutput", - "type": "object" - }, - "FileDestination": { - "additionalProperties": false, - "description": "Destination used by a file-loading step.", - "properties": { - "type": { - "description": "Variable scope.", - "enum": [ - "local", - "global" - ], - "examples": [ - "local" - ], - "title": "Type", - "type": "string" - }, - "variable": { - "description": "Destination variable name.", - "examples": [ - "selected_file" - ], - "title": "Variable", - "type": "string" - } - }, - "required": [ - "type", - "variable" - ], - "title": "FileDestination", - "type": "object" - }, - "GlobalInput": { - "additionalProperties": false, - "description": "Reads a recipe-global variable.", - "properties": { - "global_name": { - "description": "Global variable name.", - "examples": [ - "global_value" - ], - "title": "Global Name", - "type": "string" - }, - "type": { - "const": "global", - "description": "Input source type.", - "examples": [ - "global" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "global_name" - ], - "title": "GlobalInput", - "type": "object" - }, - "GlobalOutput": { - "additionalProperties": false, - "description": "Stores the output in a recipe-global variable.", - "properties": { - "global_name": { - "description": "Global destination variable.", - "examples": [ - "saved" - ], - "title": "Global Name", - "type": "string" - }, - "type": { - "const": "global", - "description": "Output mapping type.", - "examples": [ - "global" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "global_name" - ], - "title": "GlobalOutput", - "type": "object" - }, - "ImageOutput": { - "additionalProperties": false, - "description": "Publishes an image output for presentation.", - "properties": { - "type": { - "const": "image", - "description": "Output mapping type.", - "examples": [ - "image" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type" - ], - "title": "ImageOutput", - "type": "object" - }, - "InputMapping": { - "discriminator": { - "mapping": { - "direct": "#/$defs/DirectInput", - "global": "#/$defs/GlobalInput", - "local": "#/$defs/LocalInput", - "method": "#/$defs/MethodInput" - }, - "propertyName": "type" - }, - "oneOf": [ - { - "$ref": "#/$defs/DirectInput" - }, - { - "$ref": "#/$defs/LocalInput" - }, - { - "$ref": "#/$defs/GlobalInput" - }, - { - "$ref": "#/$defs/MethodInput" - } - ] - }, - "InternalSequenceReference": { - "additionalProperties": false, - "description": "Reference to another sequence in this recipe.", - "properties": { - "name": { - "description": "Target sequence name.", - "examples": [ - "Calibration" - ], - "title": "Name", - "type": "string" - }, - "type": { - "const": "internal", - "description": "Reference kind.", - "examples": [ - "internal" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "name" - ], - "title": "InternalSequenceReference", - "type": "object" - }, - "LocalInput": { - "additionalProperties": false, - "description": "Reads a sequence-local variable.", - "properties": { - "local_name": { - "description": "Local variable name.", - "examples": [ - "local_value" - ], - "title": "Local Name", - "type": "string" - }, - "type": { - "const": "local", - "description": "Input source type.", - "examples": [ - "local" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "local_name" - ], - "title": "LocalInput", - "type": "object" - }, - "LocalOutput": { - "additionalProperties": false, - "description": "Stores the output in a sequence-local variable.", - "properties": { - "local_name": { - "description": "Local destination variable.", - "examples": [ - "saved" - ], - "title": "Local Name", - "type": "string" - }, - "type": { - "const": "local", - "description": "Output mapping type.", - "examples": [ - "local" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "local_name" - ], - "title": "LocalOutput", - "type": "object" - }, - "MethodInput": { - "additionalProperties": false, - "description": "Resolves a method reference for the step.", - "properties": { - "type": { - "const": "method", - "description": "Input source type.", - "examples": [ - "method" - ], - "title": "Type", - "type": "string" - }, - "value": { - "description": "Method reference.", - "examples": [ - "helper" - ], - "title": "Value" - } - }, - "required": [ - "type", - "value" - ], - "title": "MethodInput", - "type": "object" - }, - "OutputMapping": { - "discriminator": { - "mapping": { - "equals": "#/$defs/EqualsOutput", - "global": "#/$defs/GlobalOutput", - "image": "#/$defs/ImageOutput", - "local": "#/$defs/LocalOutput", - "passfail": "#/$defs/PassFailOutput", - "passthrough": "#/$defs/PassthroughOutput", - "range": "#/$defs/RangeOutput" - }, - "propertyName": "type" - }, - "oneOf": [ - { - "$ref": "#/$defs/PassFailOutput" - }, - { - "$ref": "#/$defs/EqualsOutput" - }, - { - "$ref": "#/$defs/RangeOutput" - }, - { - "$ref": "#/$defs/PassthroughOutput" - }, - { - "$ref": "#/$defs/LocalOutput" - }, - { - "$ref": "#/$defs/GlobalOutput" - }, - { - "$ref": "#/$defs/ImageOutput" - } - ] - }, - "PassFailOutput": { - "additionalProperties": false, - "description": "Interprets the output as a pass/fail verdict.", - "properties": { - "type": { - "const": "passfail", - "description": "Output mapping type.", - "examples": [ - "passfail" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type" - ], - "title": "PassFailOutput", - "type": "object" - }, - "PassthroughOutput": { - "additionalProperties": false, - "description": "Uses the nested result without adding a verdict.", - "properties": { - "type": { - "const": "passthrough", - "description": "Output mapping type.", - "examples": [ - "passthrough" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type" - ], - "title": "PassthroughOutput", - "type": "object" - }, - "PythonModuleStep": { - "additionalProperties": false, - "description": "Calls a method or reads/writes an attribute in a Python module.", - "properties": { - "action_type": { - "description": "Operation performed on the Python module.", - "enum": [ - "method", - "read_attribute", - "write_attribute" - ], - "examples": [ - "method" - ], - "title": "Action Type", - "type": "string" - }, - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "method_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Method name for method actions.", - "examples": [ - "run" - ], - "title": "Method Name" - }, - "module": { - "description": "Python module path.", - "examples": [ - "tests.py" - ], - "title": "Module", - "type": "string" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "PythonModuleStep", - "description": "Canonical registered step type.", - "examples": [ - "PythonModuleStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype", - "action_type", - "module" - ], - "title": "PythonModuleStep", - "type": "object" - }, - "RangeOutput": { - "additionalProperties": false, - "description": "Passes when the output is within an inclusive range.", - "properties": { - "max": { - "description": "Maximum accepted value.", - "examples": [ - 4 - ], - "title": "Max" - }, - "min": { - "description": "Minimum accepted value.", - "examples": [ - 1 - ], - "title": "Min" - }, - "type": { - "const": "range", - "description": "Output mapping type.", - "examples": [ - "range" - ], - "title": "Type", - "type": "string" - } - }, - "required": [ - "type", - "min", - "max" - ], - "title": "RangeOutput", - "type": "object" - }, - "RecipeHeader": { - "additionalProperties": false, - "description": "The first YAML document, identifying a recipe and its entry sequence.", - "properties": { - "continue_on_error": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Recipe-wide error policy.", - "examples": [ - false - ], - "title": "Continue On Error" - }, - "description": { - "description": "Purpose of the recipe.", - "examples": [ - "Acceptance tests." - ], - "title": "Description", - "type": "string" - }, - "globals": { - "additionalProperties": true, - "description": "Recipe-wide variables.", - "examples": [ - {} - ], - "title": "Globals", - "type": "object" - }, - "main_sequence": { - "description": "Sequence where execution begins.", - "examples": [ - "Main" - ], - "title": "Main Sequence", - "type": "string" - }, - "name": { - "description": "Human-readable recipe name.", - "examples": [ - "Hardware acceptance" - ], - "title": "Name", - "type": "string" - }, - "recipe_version": { - "const": "2.0.0", - "description": "Version of the recipe language contract.", - "examples": [ - "2.0.0" - ], - "title": "Recipe Version", - "type": "string" - }, - "report": { - "default": "overwrite", - "description": "Report file mode.", - "enum": [ - "overwrite", - "append" - ], - "examples": [ - "overwrite" - ], - "title": "Report", - "type": "string" - }, - "report_name_include_serial": { - "default": false, - "description": "Include the serial number in the report name.", - "examples": [ - false - ], - "title": "Report Name Include Serial", - "type": "boolean" - }, - "test_package": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Package containing recipe test modules.", - "examples": [ - "acceptance" - ], - "title": "Test Package" - }, - "version": { - "description": "Version of this recipe.", - "examples": [ - "1.0" - ], - "title": "Version", - "type": "string" - } - }, - "required": [ - "name", - "version", - "recipe_version", - "description", - "main_sequence", - "globals" - ], - "title": "RecipeHeader", - "type": "object" - }, - "SSHCloseStep": { - "additionalProperties": false, - "description": "Closes the SSH client stored in recipe globals.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "SSHCloseStep", - "description": "Canonical registered step type.", - "examples": [ - "SSHCloseStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "SSHCloseStep", - "type": "object" - }, - "SSHConnectStep": { - "additionalProperties": false, - "description": "Opens the SSH client stored in recipe globals.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "SSHConnectStep", - "description": "Canonical registered step type.", - "examples": [ - "SSHConnectStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "SSHConnectStep", - "type": "object" - }, - "SSHUploadStep": { - "additionalProperties": false, - "description": "Uploads files through an SSH connection.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "files": { - "description": "Local and remote file pairs to upload.", - "examples": [ - [ - { - "local": "bin/tool", - "remote": "/tmp/tool" - } - ] - ], - "items": { - "$ref": "#/$defs/UploadFile" - }, - "title": "Files", - "type": "array" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "local_package": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional package containing local resources.", - "examples": [ - "my_package" - ], - "title": "Local Package" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "permissions": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional remote permissions.", - "examples": [ - "0755" - ], - "title": "Permissions" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "skip_if_sha256_match": { - "default": false, - "description": "Skip files whose remote checksum matches.", - "examples": [ - false - ], - "title": "Skip If Sha256 Match", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "SSHUploadStep", - "description": "Canonical registered step type.", - "examples": [ - "SSHUploadStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype", - "files" - ], - "title": "SSHUploadStep", - "type": "object" - }, - "Sequence": { - "additionalProperties": false, - "description": "One named executable sequence document.", - "properties": { - "description": { - "description": "Purpose of the sequence.", - "examples": [ - "Main sequence." - ], - "title": "Description", - "type": "string" - }, - "locals": { - "additionalProperties": true, - "description": "Variables local to the sequence.", - "examples": [ - {} - ], - "title": "Locals", - "type": "object" - }, - "outputs": { - "additionalProperties": true, - "description": "Reserved sequence output metadata.", - "examples": [ - {} - ], - "title": "Outputs", - "type": "object" - }, - "parameters": { - "additionalProperties": true, - "description": "Reserved sequence input metadata.", - "examples": [ - {} - ], - "title": "Parameters", - "type": "object" - }, - "sequence_name": { - "description": "Unique sequence name.", - "examples": [ - "Main" - ], - "title": "Sequence Name", - "type": "string" - }, - "setup_steps": { - "description": "Steps run before the main steps.", - "examples": [ - [] - ], - "items": { - "$ref": "#/$defs/Step" - }, - "title": "Setup Steps", - "type": "array" - }, - "steps": { - "description": "Ordered main steps.", - "examples": [ - [] - ], - "items": { - "$ref": "#/$defs/Step" - }, - "title": "Steps", - "type": "array" - }, - "teardown_steps": { - "description": "Steps run during teardown.", - "examples": [ - [] - ], - "items": { - "$ref": "#/$defs/Step" - }, - "title": "Teardown Steps", - "type": "array" - } - }, - "required": [ - "sequence_name", - "description", - "parameters", - "outputs", - "locals", - "setup_steps", - "steps", - "teardown_steps" - ], - "title": "Sequence", - "type": "object" - }, - "SequenceStep": { - "additionalProperties": false, - "description": "Runs another sequence as a step.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "sequence": { - "$ref": "#/$defs/InternalSequenceReference", - "description": "Internal sequence reference.", - "examples": [ - { - "name": "Calibration", - "type": "internal" - } - ] - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "SequenceStep", - "description": "Canonical registered step type.", - "examples": [ - "SequenceStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype", - "sequence" - ], - "title": "SequenceStep", - "type": "object" - }, - "SerialNumberStep": { - "additionalProperties": false, - "description": "Captures the device serial number.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "SerialNumberStep", - "description": "Canonical registered step type.", - "examples": [ - "SerialNumberStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "SerialNumberStep", - "type": "object" - }, - "Step": { - "discriminator": { - "mapping": { - "PythonModuleStep": "#/$defs/PythonModuleStep", - "SSHCloseStep": "#/$defs/SSHCloseStep", - "SSHConnectStep": "#/$defs/SSHConnectStep", - "SSHUploadStep": "#/$defs/SSHUploadStep", - "SequenceStep": "#/$defs/SequenceStep", - "SerialNumberStep": "#/$defs/SerialNumberStep", - "UserInteractionStep": "#/$defs/UserInteractionStep", - "UserLoadingStep": "#/$defs/UserLoadingStep", - "UserRunMethodStep": "#/$defs/UserRunMethodStep", - "UserWriteStep": "#/$defs/UserWriteStep", - "WaitStep": "#/$defs/WaitStep" - }, - "propertyName": "steptype" - }, - "oneOf": [ - { - "$ref": "#/$defs/PythonModuleStep" - }, - { - "$ref": "#/$defs/SequenceStep" - }, - { - "$ref": "#/$defs/UserInteractionStep" - }, - { - "$ref": "#/$defs/WaitStep" - }, - { - "$ref": "#/$defs/UserLoadingStep" - }, - { - "$ref": "#/$defs/UserRunMethodStep" - }, - { - "$ref": "#/$defs/UserWriteStep" - }, - { - "$ref": "#/$defs/SerialNumberStep" - }, - { - "$ref": "#/$defs/SSHConnectStep" - }, - { - "$ref": "#/$defs/SSHCloseStep" - }, - { - "$ref": "#/$defs/SSHUploadStep" - } - ] - }, - "UploadFile": { - "additionalProperties": false, - "description": "One local-to-remote SSH upload pair.", - "properties": { - "local": { - "description": "Local file or package resource.", - "examples": [ - "bin/tool" - ], - "title": "Local", - "type": "string" - }, - "remote": { - "description": "Remote destination path.", - "examples": [ - "/tmp/tool" - ], - "title": "Remote", - "type": "string" - } - }, - "required": [ - "local", - "remote" - ], - "title": "UploadFile", - "type": "object" - }, - "UserInteractionStep": { - "additionalProperties": false, - "description": "Displays an operator interaction prompt.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "UserInteractionStep", - "description": "Canonical registered step type.", - "examples": [ - "UserInteractionStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "UserInteractionStep", - "type": "object" - }, - "UserLoadingStep": { - "additionalProperties": false, - "description": "Prompts the operator to select a file.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "file_save_location": { - "anyOf": [ - { - "$ref": "#/$defs/FileDestination" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Local or global destination for the selected file.", - "examples": [ - { - "type": "local", - "variable": "selected_file" - } - ] - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "UserLoadingStep", - "description": "Canonical registered step type.", - "examples": [ - "UserLoadingStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "UserLoadingStep", - "type": "object" - }, - "UserRunMethodStep": { - "additionalProperties": false, - "description": "Optionally runs a Python method after an operator response.", - "properties": { - "action_type": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional Python action type.", - "examples": [ - "method" - ], - "title": "Action Type" - }, - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "method_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional Python method name.", - "examples": [ - "run" - ], - "title": "Method Name" - }, - "module": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional Python module path.", - "examples": [ - "tests.py" - ], - "title": "Module" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "UserRunMethodStep", - "description": "Canonical registered step type.", - "examples": [ - "UserRunMethodStep" - ], - "title": "Steptype", - "type": "string" - }, - "trigger_response": { - "anyOf": [ - { - "type": "string" - }, - { - "items": {}, - "type": "array" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Operator response that triggers execution.", - "examples": [ - "run" - ], - "title": "Trigger Response" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "UserRunMethodStep", - "type": "object" - }, - "UserWriteStep": { - "additionalProperties": false, - "description": "Writes an operator-provided value to a configured destination.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "UserWriteStep", - "description": "Canonical registered step type.", - "examples": [ - "UserWriteStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "UserWriteStep", - "type": "object" - }, - "WaitStep": { - "additionalProperties": false, - "description": "Waits for a non-negative duration in seconds.", - "properties": { - "continue_on_error": { - "default": false, - "description": "Per-step error policy.", - "examples": [ - false - ], - "title": "Continue On Error", - "type": "boolean" - }, - "critical": { - "default": false, - "description": "Stop on error when policy permits continuation.", - "examples": [ - false - ], - "title": "Critical", - "type": "boolean" - }, - "description": { - "description": "Purpose of the step.", - "examples": [ - "Run a test operation." - ], - "title": "Description", - "type": "string" - }, - "id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Optional stable step identifier.", - "examples": [ - "test-1" - ], - "title": "Id" - }, - "input_mapping": { - "additionalProperties": { - "$ref": "#/$defs/InputMapping" - }, - "description": "Named input sources.", - "examples": [ - {} - ], - "title": "Input Mapping", - "type": "object" - }, - "output_mapping": { - "additionalProperties": { - "$ref": "#/$defs/OutputMapping" - }, - "description": "Named verdicts and destinations.", - "examples": [ - {} - ], - "title": "Output Mapping", - "type": "object" - }, - "skip": { - "default": false, - "description": "Skip execution.", - "examples": [ - false - ], - "title": "Skip", - "type": "boolean" - }, - "step_name": { - "description": "Human-readable step name.", - "examples": [ - "Run test" - ], - "title": "Step Name", - "type": "string" - }, - "steptype": { - "const": "WaitStep", - "description": "Canonical registered step type.", - "examples": [ - "WaitStep" - ], - "title": "Steptype", - "type": "string" - } - }, - "required": [ - "step_name", - "description", - "steptype" - ], - "title": "WaitStep", - "type": "object" - } - }, - "additionalProperties": false, - "description": "Aggregate typed recipe used by tooling and JSON Schema consumers.", - "properties": { - "header": { - "$ref": "#/$defs/RecipeHeader", - "description": "Recipe header document." - }, - "sequences": { - "description": "Sequence documents.", - "items": { - "$ref": "#/$defs/Sequence" - }, - "minItems": 1, - "title": "Sequences", - "type": "array" - } - }, - "required": [ - "header", - "sequences" - ], - "title": "Recipe", - "type": "object" -} diff --git a/docs/source/conf.py b/docs/source/conf.py index 4d52a21..0e4b996 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,16 +3,20 @@ # SPDX-License-Identifier: LGPL-2.1-or-later import datetime -import importlib.util +import sys +from pathlib import Path from pypts._version import __version__ +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) + project = "pypts" author = "Alvaro Martinez Landete" version = __version__ -copyright = "{0}, CERN".format(datetime.datetime.now().year) +copyright = f"{datetime.datetime.now(datetime.UTC).year}, CERN" # -- General configuration ---------------------------------------------------- @@ -26,8 +30,6 @@ 'sphinx.ext.doctest', 'sphinx.ext.napoleon', ] -if importlib.util.find_spec("acc_py_sphinx") is not None: - extensions.insert(0, "acc_py_sphinx.theme") # Add any paths that contain templates here, relative to this directory. @@ -44,7 +46,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "acc_py" if "acc_py_sphinx.theme" in extensions else "alabaster" +html_theme = "alabaster" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -54,6 +56,22 @@ html_show_sourcelink = True +def _generate_recipe_language_docs(app): + """Generate the schema and its human reference for this Sphinx build.""" + from spikes.recipe_pydantic.artifacts import write_artifacts + + generated = Path(app.srcdir) / "_generated" + write_artifacts( + generated / "recipe_language.schema.json", + generated / "recipe_language_reference.rst", + ) + + +def setup(app): + app.connect("builder-inited", _generate_recipe_language_docs) + return {"parallel_read_safe": True, "parallel_write_safe": True} + + # -- Options for sphinx.ext.autosummary autosummary_generate = True diff --git a/docs/source/dependency_license_analysis.rst b/docs/source/dependency_license_analysis.rst index acbaa07..b471882 100644 --- a/docs/source/dependency_license_analysis.rst +++ b/docs/source/dependency_license_analysis.rst @@ -28,7 +28,6 @@ Key Findings * ✅ **100% Compatible**: All core dependencies use LGPL-compatible licenses * ✅ **No Conflicts**: No GPL-only or restrictive copyleft licenses detected * ✅ **Qt Alignment**: PySide6 usage aligns perfectly with LGPL choice -* ⚠️ **One Unknown**: acc-py-sphinx license needs verification (docs-only dependency) Core Dependencies Analysis ========================== @@ -174,10 +173,10 @@ Documentation Dependencies (``doc`` extra) - BSD-2-Clause - ✅ Yes - Documentation generator - * - acc-py-sphinx - - Unknown - - ⚠️ Verify - - CERN-specific Sphinx extension + * - Pydantic + - MIT + - ✅ Yes + - Generates the recipe schema during documentation builds Development Dependencies (``dev`` extra) ----------------------------------------- @@ -246,7 +245,6 @@ All identified dependencies fall into these categories: :Permissive Licenses (MIT, BSD): Compatible without restrictions :LGPL Libraries: Direct compatibility :GPL Libraries: Compatible (PySide6 offers LGPL option) -:Unknown Licenses: Only acc-py-sphinx (documentation only) Risk Assessment =============== @@ -260,13 +258,13 @@ Risk Assessment - Percentage - Description * - **No Risk** - - 7/8 - - 87.5% - - Core dependencies with confirmed compatible licenses + - 8/8 + - 100% + - Dependencies with confirmed compatible licenses * - **Low Risk** - - 1/8 - - 12.5% - - acc-py-sphinx (docs-only, likely permissive) + - 0/8 + - 0% + - No dependencies in this category * - **Medium Risk** - 0/8 - 0% @@ -283,8 +281,7 @@ Immediate Actions ----------------- 1. ✅ **Continue with LGPL-2.1-or-later** - All dependencies are compatible -2. ⚠️ **Verify acc-py-sphinx license** - Check CERN's repository for license information -3. 📋 **Add license attribution** - Include dependency licenses in distribution +2. 📋 **Add license attribution** - Include dependency licenses in distribution Future Monitoring ------------------ @@ -305,7 +302,7 @@ Compliance Checklist - Notes * - All dependencies LGPL-compatible - ✅ Met - - Confirmed for 7/8 dependencies + - Confirmed for 8/8 dependencies * - No GPL-only dependencies - ✅ Met - PySide6 offers LGPL option diff --git a/docs/source/index.rst b/docs/source/index.rst index 3ba2de7..59b44d1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,7 +57,7 @@ Documentation contents api architecture recipe_language_architecture - recipe_language_reference + _generated/recipe_language_reference gui_architecture yaml_format instruments diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index 45d093b..8e40c2c 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -15,9 +15,9 @@ Recipe Language 2 Architecture guidance. Exact version 2 fields, types, defaults, and examples are in the generated -:doc:`recipe_language_reference`. The aggregate schema is also available as +:doc:`_generated/recipe_language_reference`. The aggregate schema is also available as :download:`recipe_language.schema.json -<_static/recipe_language.schema.json>`. +<_generated/recipe_language.schema.json>`. Design and dependency direction ------------------------------- @@ -44,16 +44,17 @@ The modules and artifacts have deliberately one-way dependencies:: | JSON-only RST renderer | | v v - typed Recipe recipe_language_reference.rst + typed Recipe generated reference RST | | v v - future runtime adapter Sphinx + recipe.py runtime Sphinx + construction ``models.py`` never imports YAML, runtime classes, concrete steps, YamVIEW, or Sphinx. ``parser.py`` depends on the models and PyYAML, but still does not import runtime or UI code. Documentation generation reads the model only to -create JSON Schema; the RST renderer reads the committed JSON file and has no -Pydantic or Sphinx dependency. +create JSON Schema; the RST renderer reads the JSON generated for the current +build and has no Pydantic or Sphinx dependency. Parsing and information flow ---------------------------- @@ -65,27 +66,29 @@ semantics therefore remain separate stages:: recipe YAML text/file | v - SafeLoader composition --------> YAML node/path/source-span index - | | - +--> duplicate key / alias checks | - | | - v | - safe Python documents | - | | - v | - strict Pydantic models | - | | - v | - cross-document semantic pass | - | | - +--> diagnostics <-----------------+ - | code, path, severity, source, span - v + PyYAML YAML front end + | source information + +--> compose_all(SafeLoader) --> node/path/span index -----+ + | | | + | +--> duplicate/recursive-alias diagnostics ------+ + | + +--> safe_load_all() --> safe Python documents | + | | + v | + strict Pydantic models | + | | + v | + cross-document semantic pass | + | | + +--> diagnostics <---------+ + | code, path, severity, + | source, nearest span + v frozen aggregate Recipe | +--> canonical multi-document YAML | - +--> future sequencer/runtime adapter + +--> future recipe.py runtime construction ``parse_recipe_text`` and ``parse_recipe_file`` return ``ParseResult``. A valid result owns an aggregate :ref:`recipe-v2-header` plus one or more @@ -93,6 +96,14 @@ valid result owns an aggregate :ref:`recipe-v2-header` plus one or more diagnostic tuple when errors exist. ``dump_recipe`` writes canonical version 2 YAML; comments and original formatting are not a round-trip guarantee. +Here, "composition" is PyYAML terminology, not a PyPTS adapter or an additional +recipe representation. ``yaml.compose_all(..., Loader=yaml.SafeLoader)`` +returns PyYAML node objects with source marks, which the parser uses to detect +duplicate keys and recursive aliases and to index diagnostic spans. +``yaml.safe_load_all()`` separately constructs ordinary safe Python values for +Pydantic. Both operations use PyYAML's safe loader; neither constructs runtime +``Recipe`` or ``Step`` objects. + Structural and custom semantic rules ------------------------------------ @@ -178,7 +189,7 @@ values. The intended editor flow is:: - committed JSON Schema + generated/published JSON Schema | +--> discriminator choices --> step/mapping selectors | @@ -211,20 +222,50 @@ aggregate typed model, not raw YAML or loosely typed dictionaries:: frozen Recipe model | v - runtime adapter - | | | - | | +--> typed input/output mapping adapters - | +-------> concrete runtime step construction - +------------> sequence table and nested reference binding - | - v + recipe.py: Recipe + | | + | +-------> sequence table and nested reference binding + v + recipe.py: Sequence + | + v + Step.build_step() for each typed definition + | + v + STEP_TYPE_REGISTRY --> concrete classes in steps.py + | + v setup_steps -> steps -> teardown_steps -The adapter will translate language models to runtime objects exactly once. -It must not reparse YAML, repeat structural validation, or keep another -supported-type registry. Invalid recipes never instantiate concrete runtime -steps. Runtime-only behavior—execution events, error policy, reports, hardware -access, and GUI interaction—stays downstream of the language model. +This is an evolution of the runtime construction that already exists in +``recipe.py``; it is not a separate adapter module. Today ``Recipe`` loads raw +YAML documents, ``Sequence`` iterates step dictionaries, and +``Step.build_step()`` validates each dictionary before selecting an executable +class from ``steps.py``. The integration changes the input to that path: +``Recipe`` receives the validated aggregate model, ``Sequence`` iterates typed +definitions, and ``Step.build_step()`` becomes a small typed factory. + +The runtime registry remains because a canonical discriminator such as +``PythonModuleStep`` must be associated with the Python class that implements +its behavior. It is a behavior registry, not a second language schema: field +names, types, defaults, and structural rules remain exclusively in the +Pydantic models. A completeness test will require every authorable model +discriminator to have exactly one executable implementation. + +Concrete ``_step()`` methods in ``steps.py`` continue to own execution. For +example, the executable ``PythonModuleStep`` still imports and invokes Python +code; it no longer needs to validate an untrusted recipe dictionary. Common +definition fields can be passed through one base ``Step.from_definition()`` +implementation, with concrete overrides only when runtime state differs from +authored data. ``IndexedStep`` remains a runtime-generated wrapper and is +never added to the authorable model union. + +Synthetic runtime operations are also constructed directly. For example, +``Recipe.run()`` must not fabricate a recipe dictionary merely to execute the +main sequence. No runtime construction reparses YAML or repeats Pydantic +structural validation, and invalid recipes never instantiate executable steps. +Execution events, error policy, reports, hardware access, and GUI interaction +remain downstream of the frozen language model. Canonical documentation recipe ------------------------------ @@ -242,25 +283,40 @@ spike. It is not a production bundled recipe. Maintaining the documentation ----------------------------- -The tracked artifacts have final paths under ``docs/source``. A normal Sphinx -build reads them directly and performs no generation or copying:: - - python -m spikes.recipe_pydantic.artifacts - -The command writes ``_static/recipe_language.schema.json`` from Pydantic and -then writes ``recipe_language_reference.rst`` by parsing that JSON. Check that -both committed files are current without modifying them:: - - python -m spikes.recipe_pydantic.artifacts --check +Every Sphinx build generates both artifacts before reading documentation +sources:: + + Pydantic models + | + v + _generated/recipe_language.schema.json + | + v + JSON-only RST renderer + | + v + _generated/recipe_language_reference.rst + +The Sphinx ``builder-inited`` hook writes these files to an ignored staging +directory under ``docs/source``. The schema is copied into the HTML output as a +download, and the generated RST is included in the toctree. Neither generated +file is maintained manually or treated as a committed source artifact. + +The documentation environment therefore installs Pydantic. CI and any +documentation build image must install the ``doc`` extra and include the model, +schema generator, and JSON-only renderer sources. This is a build-time +dependency for Phase 5; it does not make Pydantic a production runtime +dependency by itself. When adding a step or mapping, update its Pydantic model and discriminated -union, add an independent round-trip fixture, regenerate both artifacts, and -review the schema and RST diffs. Add custom semantic code only when a rule -requires document, sibling, or ordering context. Handwritten architecture -prose explains those relationships; it must link to generated fields rather -than restating field tables. - -The documentation contract is protected by tests that compare the model to the -committed JSON, compare the JSON-only renderer to the committed RST, count all -discriminator variants, validate the example, check literal-include markers, -and build Sphinx with warnings treated as errors. +union, add an independent round-trip fixture, and run the documentation build. +Review the generated schema and reference in the build output when relevant. +Add custom semantic code only when a rule requires document, sibling, or +ordering context. Handwritten architecture prose explains those relationships; +it must link to generated fields rather than restating field tables. + +The documentation contract is protected by tests that generate into temporary +directories, verify deterministic model-to-JSON and JSON-to-RST output, count +all discriminator variants, validate the example, check literal-include +markers, and build Sphinx with warnings treated as errors. Generation failure +therefore fails the same build that would publish the documentation. diff --git a/docs/source/recipe_language_reference.rst b/docs/source/recipe_language_reference.rst deleted file mode 100644 index ac7f8b5..0000000 --- a/docs/source/recipe_language_reference.rst +++ /dev/null @@ -1,784 +0,0 @@ -.. SPDX-FileCopyrightText: 2026 CERN -.. -.. SPDX-License-Identifier: CC-BY-SA-4.0 -.. -.. Generated from recipe_language.schema.json. Do not edit manually. - -Recipe Language 2.0 Reference -============================= - -This page is generated from the tracked aggregate JSON Schema. It describes -the accepted future recipe language model; production execution still uses -the version 1 language until Phase 6 integration is complete. - -:download:`Download the JSON Schema <_static/recipe_language.schema.json>`. - -See :doc:`recipe_language_architecture` for parsing, semantic rules, -documentation maintenance, and the planned YamVIEW and sequencer flows. - -Documents ---------- - -.. _recipe-v2-header: - -RecipeHeader -~~~~~~~~~~~~ - -The first YAML document, identifying a recipe and its entry sequence. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``continue_on_error`` - - ``bool | None`` - - optional; default ``null`` - - Recipe-wide error policy. Example: ``false``. - * - ``description`` - - ``str`` - - required - - Purpose of the recipe. Example: ``"Acceptance tests."``. - * - ``globals`` - - ``object`` - - required - - Recipe-wide variables. Example: ``{}``. - * - ``main_sequence`` - - ``str`` - - required - - Sequence where execution begins. Example: ``"Main"``. - * - ``name`` - - ``str`` - - required - - Human-readable recipe name. Example: ``"Hardware acceptance"``. - * - ``recipe_version`` - - ``'2.0.0'`` - - required - - Version of the recipe language contract. Example: ``"2.0.0"``. - * - ``report`` - - ``'overwrite' | 'append'`` - - optional; default ``"overwrite"`` - - Report file mode. Example: ``"overwrite"``. - * - ``report_name_include_serial`` - - ``bool`` - - optional; default ``false`` - - Include the serial number in the report name. Example: ``false``. - * - ``test_package`` - - ``str | None`` - - optional; default ``null`` - - Package containing recipe test modules. Example: ``"acceptance"``. - * - ``version`` - - ``str`` - - required - - Version of this recipe. Example: ``"1.0"``. - -.. _recipe-v2-sequence: - -Sequence -~~~~~~~~ - -One named executable sequence document. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``description`` - - ``str`` - - required - - Purpose of the sequence. Example: ``"Main sequence."``. - * - ``locals`` - - ``object`` - - required - - Variables local to the sequence. Example: ``{}``. - * - ``outputs`` - - ``object`` - - required - - Reserved sequence output metadata. Example: ``{}``. - * - ``parameters`` - - ``object`` - - required - - Reserved sequence input metadata. Example: ``{}``. - * - ``sequence_name`` - - ``str`` - - required - - Unique sequence name. Example: ``"Main"``. - * - ``setup_steps`` - - ``list[Step]`` - - required - - Steps run before the main steps. Example: ``[]``. - * - ``steps`` - - ``list[Step]`` - - required - - Ordered main steps. Example: ``[]``. - * - ``teardown_steps`` - - ``list[Step]`` - - required - - Steps run during teardown. Example: ``[]``. - -Nested structures ------------------ - -.. _recipe-v2-structure-internalsequencereference: - -InternalSequenceReference -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Reference to another sequence in this recipe. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``name`` - - ``str`` - - required - - Target sequence name. Example: ``"Calibration"``. - * - ``type`` - - ``'internal'`` - - required - - Reference kind. Example: ``"internal"``. - -.. _recipe-v2-structure-filedestination: - -FileDestination -~~~~~~~~~~~~~~~ - -Destination used by a file-loading step. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'local' | 'global'`` - - required - - Variable scope. Example: ``"local"``. - * - ``variable`` - - ``str`` - - required - - Destination variable name. Example: ``"selected_file"``. - -.. _recipe-v2-structure-uploadfile: - -UploadFile -~~~~~~~~~~ - -One local-to-remote SSH upload pair. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``local`` - - ``str`` - - required - - Local file or package resource. Example: ``"bin/tool"``. - * - ``remote`` - - ``str`` - - required - - Remote destination path. Example: ``"/tmp/tool"``. - -Common step fields ------------------- - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``continue_on_error`` - - ``bool`` - - optional; default ``false`` - - Per-step error policy. Example: ``false``. - * - ``critical`` - - ``bool`` - - optional; default ``false`` - - Stop on error when policy permits continuation. Example: ``false``. - * - ``description`` - - ``str`` - - required - - Purpose of the step. Example: ``"Run a test operation."``. - * - ``id`` - - ``str | None`` - - optional; default ``null`` - - Optional stable step identifier. Example: ``"test-1"``. - * - ``input_mapping`` - - ``dict[str, InputMapping]`` - - optional - - Named input sources. Example: ``{}``. - * - ``output_mapping`` - - ``dict[str, OutputMapping]`` - - optional - - Named verdicts and destinations. Example: ``{}``. - * - ``skip`` - - ``bool`` - - optional; default ``false`` - - Skip execution. Example: ``false``. - * - ``step_name`` - - ``str`` - - required - - Human-readable step name. Example: ``"Run test"``. - -Authorable steps ----------------- - -.. _recipe-v2-step-pythonmodulestep: - -PythonModuleStep -~~~~~~~~~~~~~~~~ - -Calls a method or reads/writes an attribute in a Python module. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``action_type`` - - ``'method' | 'read_attribute' | 'write_attribute'`` - - required - - Operation performed on the Python module. Example: ``"method"``. - * - ``method_name`` - - ``str | None`` - - optional; default ``null`` - - Method name for method actions. Example: ``"run"``. - * - ``module`` - - ``str`` - - required - - Python module path. Example: ``"tests.py"``. - * - ``steptype`` - - ``'PythonModuleStep'`` - - required - - Canonical registered step type. Example: ``"PythonModuleStep"``. - -.. _recipe-v2-step-sshclosestep: - -SSHCloseStep -~~~~~~~~~~~~ - -Closes the SSH client stored in recipe globals. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'SSHCloseStep'`` - - required - - Canonical registered step type. Example: ``"SSHCloseStep"``. - -.. _recipe-v2-step-sshconnectstep: - -SSHConnectStep -~~~~~~~~~~~~~~ - -Opens the SSH client stored in recipe globals. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'SSHConnectStep'`` - - required - - Canonical registered step type. Example: ``"SSHConnectStep"``. - -.. _recipe-v2-step-sshuploadstep: - -SSHUploadStep -~~~~~~~~~~~~~ - -Uploads files through an SSH connection. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``files`` - - ``list[UploadFile]`` - - required - - Local and remote file pairs to upload. Example: ``[{"local": "bin/tool", "remote": "/tmp/tool"}]``. - * - ``local_package`` - - ``str | None`` - - optional; default ``null`` - - Optional package containing local resources. Example: ``"my_package"``. - * - ``permissions`` - - ``int | str | None`` - - optional; default ``null`` - - Optional remote permissions. Example: ``"0755"``. - * - ``skip_if_sha256_match`` - - ``bool`` - - optional; default ``false`` - - Skip files whose remote checksum matches. Example: ``false``. - * - ``steptype`` - - ``'SSHUploadStep'`` - - required - - Canonical registered step type. Example: ``"SSHUploadStep"``. - -.. _recipe-v2-step-sequencestep: - -SequenceStep -~~~~~~~~~~~~ - -Runs another sequence as a step. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``sequence`` - - ``InternalSequenceReference`` - - required - - Internal sequence reference. Example: ``{"name": "Calibration", "type": "internal"}``. - * - ``steptype`` - - ``'SequenceStep'`` - - required - - Canonical registered step type. Example: ``"SequenceStep"``. - -.. _recipe-v2-step-serialnumberstep: - -SerialNumberStep -~~~~~~~~~~~~~~~~ - -Captures the device serial number. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'SerialNumberStep'`` - - required - - Canonical registered step type. Example: ``"SerialNumberStep"``. - -.. _recipe-v2-step-userinteractionstep: - -UserInteractionStep -~~~~~~~~~~~~~~~~~~~ - -Displays an operator interaction prompt. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'UserInteractionStep'`` - - required - - Canonical registered step type. Example: ``"UserInteractionStep"``. - -.. _recipe-v2-step-userloadingstep: - -UserLoadingStep -~~~~~~~~~~~~~~~ - -Prompts the operator to select a file. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``file_save_location`` - - ``FileDestination | None`` - - optional; default ``null`` - - Local or global destination for the selected file. Example: ``{"type": "local", "variable": "selected_file"}``. - * - ``steptype`` - - ``'UserLoadingStep'`` - - required - - Canonical registered step type. Example: ``"UserLoadingStep"``. - -.. _recipe-v2-step-userrunmethodstep: - -UserRunMethodStep -~~~~~~~~~~~~~~~~~ - -Optionally runs a Python method after an operator response. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``action_type`` - - ``str | None`` - - optional; default ``null`` - - Optional Python action type. Example: ``"method"``. - * - ``method_name`` - - ``str | None`` - - optional; default ``null`` - - Optional Python method name. Example: ``"run"``. - * - ``module`` - - ``str | None`` - - optional; default ``null`` - - Optional Python module path. Example: ``"tests.py"``. - * - ``steptype`` - - ``'UserRunMethodStep'`` - - required - - Canonical registered step type. Example: ``"UserRunMethodStep"``. - * - ``trigger_response`` - - ``str | list[any] | object | None`` - - optional; default ``null`` - - Operator response that triggers execution. Example: ``"run"``. - -.. _recipe-v2-step-userwritestep: - -UserWriteStep -~~~~~~~~~~~~~ - -Writes an operator-provided value to a configured destination. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'UserWriteStep'`` - - required - - Canonical registered step type. Example: ``"UserWriteStep"``. - -.. _recipe-v2-step-waitstep: - -WaitStep -~~~~~~~~ - -Waits for a non-negative duration in seconds. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``steptype`` - - ``'WaitStep'`` - - required - - Canonical registered step type. Example: ``"WaitStep"``. - -Input mappings --------------- - -.. _recipe-v2-input-direct: - -DirectInput -~~~~~~~~~~~ - -Provides a literal value. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``indexed`` - - ``bool`` - - optional; default ``false`` - - Expand a list into indexed steps. Example: ``false``. - * - ``type`` - - ``'direct'`` - - required - - Input source type. Example: ``"direct"``. - * - ``value`` - - ``any`` - - required - - Literal input value. Example: ``1``. - -.. _recipe-v2-input-global: - -GlobalInput -~~~~~~~~~~~ - -Reads a recipe-global variable. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``global_name`` - - ``str`` - - required - - Global variable name. Example: ``"global_value"``. - * - ``type`` - - ``'global'`` - - required - - Input source type. Example: ``"global"``. - -.. _recipe-v2-input-local: - -LocalInput -~~~~~~~~~~ - -Reads a sequence-local variable. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``local_name`` - - ``str`` - - required - - Local variable name. Example: ``"local_value"``. - * - ``type`` - - ``'local'`` - - required - - Input source type. Example: ``"local"``. - -.. _recipe-v2-input-method: - -MethodInput -~~~~~~~~~~~ - -Resolves a method reference for the step. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'method'`` - - required - - Input source type. Example: ``"method"``. - * - ``value`` - - ``any`` - - required - - Method reference. Example: ``"helper"``. - -Output mappings ---------------- - -.. _recipe-v2-output-equals: - -EqualsOutput -~~~~~~~~~~~~ - -Passes when the output equals the configured value. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'equals'`` - - required - - Output mapping type. Example: ``"equals"``. - * - ``value`` - - ``any`` - - required - - Expected value. Example: ``3``. - -.. _recipe-v2-output-global: - -GlobalOutput -~~~~~~~~~~~~ - -Stores the output in a recipe-global variable. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``global_name`` - - ``str`` - - required - - Global destination variable. Example: ``"saved"``. - * - ``type`` - - ``'global'`` - - required - - Output mapping type. Example: ``"global"``. - -.. _recipe-v2-output-image: - -ImageOutput -~~~~~~~~~~~ - -Publishes an image output for presentation. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'image'`` - - required - - Output mapping type. Example: ``"image"``. - -.. _recipe-v2-output-local: - -LocalOutput -~~~~~~~~~~~ - -Stores the output in a sequence-local variable. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``local_name`` - - ``str`` - - required - - Local destination variable. Example: ``"saved"``. - * - ``type`` - - ``'local'`` - - required - - Output mapping type. Example: ``"local"``. - -.. _recipe-v2-output-passfail: - -PassFailOutput -~~~~~~~~~~~~~~ - -Interprets the output as a pass/fail verdict. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'passfail'`` - - required - - Output mapping type. Example: ``"passfail"``. - -.. _recipe-v2-output-passthrough: - -PassthroughOutput -~~~~~~~~~~~~~~~~~ - -Uses the nested result without adding a verdict. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``type`` - - ``'passthrough'`` - - required - - Output mapping type. Example: ``"passthrough"``. - -.. _recipe-v2-output-range: - -RangeOutput -~~~~~~~~~~~ - -Passes when the output is within an inclusive range. - -.. list-table:: Fields - :header-rows: 1 - :widths: 18 19 25 38 - - * - Field - - Type - - Requirement - - Description and example - * - ``max`` - - ``any`` - - required - - Maximum accepted value. Example: ``4``. - * - ``min`` - - ``any`` - - required - - Minimum accepted value. Example: ``1``. - * - ``type`` - - ``'range'`` - - required - - Output mapping type. Example: ``"range"``. diff --git a/docs/source/yaml_format.rst b/docs/source/yaml_format.rst index 23e42fc..e175737 100644 --- a/docs/source/yaml_format.rst +++ b/docs/source/yaml_format.rst @@ -7,9 +7,9 @@ This page documents the recipe syntax currently accepted by the production runtime (recipe language version 1). The accepted future Pydantic design is recipe language ``2.0.0``; see :doc:`recipe_language_architecture` for its - architecture and :doc:`recipe_language_reference` for its generated syntax - reference. Do not use the version 2 example with production execution until - the integration phase is complete. + architecture and :doc:`_generated/recipe_language_reference` for its + generated syntax reference. Do not use the version 2 example with + production execution until the integration phase is complete. .. _yaml_format: diff --git a/pyproject.toml b/pyproject.toml index 4e3c72b..237993a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,8 +52,7 @@ test = [ ] doc = [ "Sphinx", - "sphinx", - "acc-py-sphinx", + "pydantic>=2.13,<3", ] dev = [ # Dependencies for development of the project, such as type checkers, @@ -62,7 +61,6 @@ dev = [ # Include the "test" and "doc" dependencies in the dev dependencies. "pts-framework[doc,test]", - "pydantic>=2.13,<3", "ruff", ] diff --git a/spikes/recipe_pydantic/artifacts.py b/spikes/recipe_pydantic/artifacts.py index e65358b..b10eca3 100644 --- a/spikes/recipe_pydantic/artifacts.py +++ b/spikes/recipe_pydantic/artifacts.py @@ -12,8 +12,9 @@ from .reference import render_reference ROOT = Path(__file__).parents[2] -DEFAULT_SCHEMA_PATH = ROOT / "docs" / "source" / "_static" / "recipe_language.schema.json" -DEFAULT_REFERENCE_PATH = ROOT / "docs" / "source" / "recipe_language_reference.rst" +DEFAULT_GENERATED_DIR = ROOT / "docs" / "source" / "_generated" +DEFAULT_SCHEMA_PATH = DEFAULT_GENERATED_DIR / "recipe_language.schema.json" +DEFAULT_REFERENCE_PATH = DEFAULT_GENERATED_DIR / "recipe_language_reference.rst" def render_json_schema() -> str: @@ -36,13 +37,14 @@ def write_artifacts( schema_path: str | Path = DEFAULT_SCHEMA_PATH, reference_path: str | Path = DEFAULT_REFERENCE_PATH, ) -> None: - schema_text, reference_text = rendered_artifacts() - for path, content in ( - (Path(schema_path), schema_text), - (Path(reference_path), reference_text), - ): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") + schema_path = Path(schema_path) + reference_path = Path(reference_path) + schema_path.parent.mkdir(parents=True, exist_ok=True) + reference_path.parent.mkdir(parents=True, exist_ok=True) + + schema_path.write_text(render_json_schema(), encoding="utf-8") + schema = json.loads(schema_path.read_text(encoding="utf-8")) + reference_path.write_text(render_reference(schema), encoding="utf-8") def check_artifacts( diff --git a/spikes/recipe_pydantic/reference.py b/spikes/recipe_pydantic/reference.py index 1879c6b..028645d 100644 --- a/spikes/recipe_pydantic/reference.py +++ b/spikes/recipe_pydantic/reference.py @@ -1,4 +1,4 @@ -"""Render the Sphinx recipe reference from committed JSON Schema only.""" +"""Render the Sphinx recipe reference from generated JSON Schema only.""" from __future__ import annotations @@ -161,13 +161,14 @@ def render_reference(schema: dict[str, Any]) -> str: "Recipe Language 2.0 Reference", "=============================", "", - "This page is generated from the tracked aggregate JSON Schema. It describes", + "This page is generated from the current build's aggregate JSON Schema. It", + "describes", "the accepted future recipe language model; production execution still uses", "the version 1 language until Phase 6 integration is complete.", "", - ":download:`Download the JSON Schema <_static/recipe_language.schema.json>`.", + ":download:`Download the JSON Schema `.", "", - "See :doc:`recipe_language_architecture` for parsing, semantic rules,", + "See :doc:`/recipe_language_architecture` for parsing, semantic rules,", "documentation maintenance, and the planned YamVIEW and sequencer flows.", "", "Documents", diff --git a/tests/unit_tests/test_recipe_pydantic_docs.py b/tests/unit_tests/test_recipe_pydantic_docs.py index 9e138aa..105472e 100644 --- a/tests/unit_tests/test_recipe_pydantic_docs.py +++ b/tests/unit_tests/test_recipe_pydantic_docs.py @@ -9,11 +9,9 @@ from pathlib import Path from spikes.recipe_pydantic.artifacts import ( - DEFAULT_REFERENCE_PATH, - DEFAULT_SCHEMA_PATH, - check_artifacts, main, render_json_schema, + rendered_artifacts, write_artifacts, ) from spikes.recipe_pydantic.parser import ( @@ -33,13 +31,12 @@ def _mapping(schema, name): return schema["$defs"][name]["discriminator"]["mapping"] -def test_committed_schema_and_reference_are_current(): - schema_text = render_json_schema() +def test_schema_and_reference_generation_is_deterministic(): + schema_text, reference = rendered_artifacts() schema = json.loads(schema_text) - assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") - assert render_reference(schema) == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") - assert check_artifacts() - assert main(["--check"]) == 0 + assert schema_text == render_json_schema() + assert reference == render_reference(schema) + assert (schema_text, reference) == rendered_artifacts() def test_check_mode_detects_stale_artifacts_without_writing(tmp_path): @@ -58,8 +55,8 @@ def test_check_mode_detects_stale_artifacts_without_writing(tmp_path): def test_every_discriminator_is_rendered_once(): - schema = json.loads(DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8")) - reference = DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + schema_text, reference = rendered_artifacts() + schema = json.loads(schema_text) for group, definition in ( ("step", "Step"), ("input", "InputMapping"), @@ -71,8 +68,8 @@ def test_every_discriminator_is_rendered_once(): def test_reference_metadata_comes_from_json_schema(): - schema = json.loads(DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8")) - reference = DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + schema_text, reference = rendered_artifacts() + schema = json.loads(schema_text) direct = schema["$defs"]["DirectInput"]["properties"]["indexed"] assert direct["description"] in reference assert f"default ``{json.dumps(direct['default'])}``" in reference @@ -146,6 +143,6 @@ def test_literalinclude_markers_are_unique_and_paired(): def test_sphinx_sources_link_reference_schema_and_example(): index = (ROOT / "docs" / "source" / "index.rst").read_text(encoding="utf-8") architecture = ARCHITECTURE.read_text(encoding="utf-8") - assert "recipe_language_reference" in index - assert "_static/recipe_language.schema.json" in architecture + assert "_generated/recipe_language_reference" in index + assert "_generated/recipe_language.schema.json" in architecture assert "_examples/recipe_v2.yml" in architecture From 7ee4f3d260097c679db34086cd91c177eef714b1 Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 10:12:50 +0200 Subject: [PATCH 08/14] migrating recipe language to pydantic-validated steps --- Dockerfile | 1 - docs/source/conf.py | 2 +- docs/source/recipe_language_architecture.rst | 79 +- docs/source/yaml_format.rst | 10 +- pyproject.toml | 2 +- spikes/recipe_pydantic/__init__.py | 48 +- spikes/recipe_pydantic/models.py | 386 ------ spikes/recipe_pydantic/parser.py | 542 --------- spikes/recipe_pydantic/reference.py | 214 ---- .../recipe_pydantic/test_recipe_pydantic.py | 410 ------- src/pypts/YamVIEW/customGUIModules.py | 22 +- src/pypts/YamVIEW/recipe_creator.py | 63 +- src/pypts/YamVIEW/recipe_rules.py | 35 - src/pypts/YamVIEW/recipe_sequencer_setup.py | 68 +- src/pypts/YamVIEW/recipe_step_setup.py | 350 +++++- src/pypts/YamVIEW/verify_recipe.py | 496 ++------ src/pypts/recipe.py | 299 ++--- .../pypts/recipe_artifacts.py | 30 +- src/pypts/recipe_language.py | 1032 ++++++----------- src/pypts/recipe_parser.py | 878 +++++++------- src/pypts/recipe_reference.py | 453 +++----- src/pypts/steps.py | 2 + tests/functional_tests/test_recipes_format.py | 5 +- tests/unit_tests/test_a_gui.py | 11 +- tests/unit_tests/test_recipe.py | 1019 +++------------- tests/unit_tests/test_recipe_language.py | 446 +++++-- tests/unit_tests/test_recipe_parser.py | 214 +--- tests/unit_tests/test_recipe_pydantic_docs.py | 12 +- tests/unit_tests/test_recipe_reference.py | 277 +---- tests/unit_tests/test_steps.py | 87 +- tests/unit_tests/test_verify_recipe.py | 215 +--- tests/unit_tests/test_yamview_schema.py | 53 + 32 files changed, 2336 insertions(+), 5425 deletions(-) delete mode 100644 spikes/recipe_pydantic/models.py delete mode 100644 spikes/recipe_pydantic/parser.py delete mode 100644 spikes/recipe_pydantic/reference.py delete mode 100644 spikes/recipe_pydantic/test_recipe_pydantic.py delete mode 100644 src/pypts/YamVIEW/recipe_rules.py rename spikes/recipe_pydantic/artifacts.py => src/pypts/recipe_artifacts.py (76%) create mode 100644 tests/unit_tests/test_yamview_schema.py diff --git a/Dockerfile b/Dockerfile index b5221ed..5adb1f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,6 @@ COPY pyproject.toml README.md ./ COPY src ./src COPY tests ./tests COPY docs ./docs -COPY spikes/recipe_pydantic ./spikes/recipe_pydantic RUN python -m pip install --no-cache-dir --upgrade pip \ && python -m pip install --no-cache-dir ".[test,doc]" build diff --git a/docs/source/conf.py b/docs/source/conf.py index 0e4b996..cded14f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,7 +58,7 @@ def _generate_recipe_language_docs(app): """Generate the schema and its human reference for this Sphinx build.""" - from spikes.recipe_pydantic.artifacts import write_artifacts + from pypts.recipe_artifacts import write_artifacts generated = Path(app.srcdir) / "_generated" write_artifacts( diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index 8e40c2c..565460c 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -7,12 +7,9 @@ Recipe Language 2 Architecture .. important:: - This page describes the accepted architecture for recipe language - ``2.0.0``. The Pydantic implementation is currently an isolated reference - under ``spikes/recipe_pydantic``. Production parsing, execution, YamVIEW, - and bundled recipes continue to use the version 1 implementation until the - integration phase is complete. See :doc:`yaml_format` for current runtime - guidance. + Recipe language ``2.0.0`` is the only production parsing and execution + path. Bundled version 1 recipes remain intentionally unmigrated and are + rejected with migration diagnostics. Exact version 2 fields, types, defaults, and examples are in the generated :doc:`_generated/recipe_language_reference`. The aggregate schema is also available as @@ -30,12 +27,12 @@ maintain another field registry. The modules and artifacts have deliberately one-way dependencies:: - models.py + pypts.recipe_language | Pydantic fields and discriminated unions +--------------------------+ | | v v - parser.py JSON Schema generator + pypts.recipe_parser JSON Schema generator | | | v | recipe_language.schema.json @@ -50,8 +47,8 @@ The modules and artifacts have deliberately one-way dependencies:: recipe.py runtime Sphinx construction -``models.py`` never imports YAML, runtime classes, concrete steps, YamVIEW, or -Sphinx. ``parser.py`` depends on the models and PyYAML, but still does not +``recipe_language`` never imports YAML, runtime classes, concrete steps, YamVIEW, or +Sphinx. ``recipe_parser`` depends on the models and PyYAML, but still does not import runtime or UI code. Documentation generation reads the model only to create JSON Schema; the RST renderer reads the JSON generated for the current build and has no Pydantic or Sphinx dependency. @@ -88,7 +85,7 @@ semantics therefore remain separate stages:: | +--> canonical multi-document YAML | - +--> future recipe.py runtime construction + +--> recipe.py runtime construction ``parse_recipe_text`` and ``parse_recipe_file`` return ``ParseResult``. A valid result owns an aggregate :ref:`recipe-v2-header` plus one or more @@ -113,7 +110,7 @@ envelope and nearest YAML span. For example, an indexed :ref:`recipe-v2-input-direct` must hold a list: -.. literalinclude:: ../../spikes/recipe_pydantic/models.py +.. literalinclude:: ../../src/pypts/recipe_language.py :language: python :start-after: # docs:indexed-direct-start :end-before: # docs:indexed-direct-end @@ -121,7 +118,7 @@ For example, an indexed :ref:`recipe-v2-input-direct` must hold a list: A Python method action requires ``method_name``: -.. literalinclude:: ../../spikes/recipe_pydantic/models.py +.. literalinclude:: ../../src/pypts/recipe_language.py :language: python :start-after: # docs:method-name-start :end-before: # docs:method-name-end @@ -129,7 +126,7 @@ A Python method action requires ``method_name``: Likewise, :ref:`recipe-v2-step-waitstep` requires a named ``wait_time`` input: -.. literalinclude:: ../../spikes/recipe_pydantic/models.py +.. literalinclude:: ../../src/pypts/recipe_language.py :language: python :start-after: # docs:wait-time-start :end-before: # docs:wait-time-end @@ -139,7 +136,7 @@ Other rules require context that no individual JSON object or JSON Schema can see. They remain in one explicit semantic pass. Sequence names must be unique and ``main_sequence`` must resolve: -.. literalinclude:: ../../spikes/recipe_pydantic/parser.py +.. literalinclude:: ../../src/pypts/recipe_parser.py :language: python :start-after: # docs:sequence-semantics-start :end-before: # docs:sequence-semantics-end @@ -148,7 +145,7 @@ unique and ``main_sequence`` must resolve: Every :ref:`recipe-v2-step-sequencestep` target is then resolved across all loaded documents: -.. literalinclude:: ../../spikes/recipe_pydantic/parser.py +.. literalinclude:: ../../src/pypts/recipe_parser.py :language: python :start-after: # docs:nested-reference-start :end-before: # docs:nested-reference-end @@ -157,7 +154,7 @@ loaded documents: Indexed lists on one step must have equal lengths, while :ref:`recipe-v2-output-passthrough` must be the only verdict-producing output: -.. literalinclude:: ../../spikes/recipe_pydantic/parser.py +.. literalinclude:: ../../src/pypts/recipe_parser.py :language: python :start-after: # docs:mapping-semantics-start :end-before: # docs:mapping-semantics-end @@ -167,7 +164,7 @@ SSH rules need both recipe globals and execution order. The semantic pass checks required connection globals and credentials, rejects an upload before a connection, and requires an opened connection to be closed: -.. literalinclude:: ../../spikes/recipe_pydantic/parser.py +.. literalinclude:: ../../src/pypts/recipe_parser.py :language: python :start-after: # docs:ssh-semantics-start :end-before: # docs:ssh-semantics-end @@ -178,16 +175,16 @@ aggregate structure, not multi-document YAML safety, source spans, equality between sibling list lengths, reference resolution, or ordered lifecycle state. -How YamVIEW will consume the language --------------------------------------- +How YamVIEW consumes the language +--------------------------------- -YamVIEW will treat the aggregate JSON Schema as its form description. The +YamVIEW treats the aggregate JSON Schema as its form description. The ``Step``, ``InputMapping``, and ``OutputMapping`` discriminator maps enumerate available variants; referenced definitions provide properties, required fields, strict types, defaults, descriptions, examples, and allowed literal values. -The intended editor flow is:: +The editor flow is:: generated/published JSON Schema | @@ -210,10 +207,10 @@ not own supported step names or field rules. Whole-recipe validation always goes through the parser so semantic rules and YAML diagnostics are identical between YamVIEW, command-line tools, and runtime loading. -How the sequencer will consume the model ----------------------------------------- +How the sequencer consumes the model +------------------------------------ -The sequencer integration begins only after parsing succeeds. It receives the +Runtime construction begins only after parsing succeeds. It receives the aggregate typed model, not raw YAML or loosely typed dictionaries:: ParseResult.require_recipe() @@ -237,27 +234,23 @@ aggregate typed model, not raw YAML or loosely typed dictionaries:: v setup_steps -> steps -> teardown_steps -This is an evolution of the runtime construction that already exists in -``recipe.py``; it is not a separate adapter module. Today ``Recipe`` loads raw -YAML documents, ``Sequence`` iterates step dictionaries, and -``Step.build_step()`` validates each dictionary before selecting an executable -class from ``steps.py``. The integration changes the input to that path: -``Recipe`` receives the validated aggregate model, ``Sequence`` iterates typed -definitions, and ``Step.build_step()`` becomes a small typed factory. +This is implemented in the existing runtime construction path, not a separate +adapter module. ``Recipe`` receives the validated aggregate model, ``Sequence`` +iterates typed definitions, and ``Step.build_step()`` is the sole typed factory +that selects an executable class from ``steps.py``. The runtime registry remains because a canonical discriminator such as ``PythonModuleStep`` must be associated with the Python class that implements its behavior. It is a behavior registry, not a second language schema: field names, types, defaults, and structural rules remain exclusively in the -Pydantic models. A completeness test will require every authorable model +Pydantic models. A completeness test requires every authorable model discriminator to have exactly one executable implementation. Concrete ``_step()`` methods in ``steps.py`` continue to own execution. For example, the executable ``PythonModuleStep`` still imports and invokes Python -code; it no longer needs to validate an untrusted recipe dictionary. Common -definition fields can be passed through one base ``Step.from_definition()`` -implementation, with concrete overrides only when runtime state differs from -authored data. ``IndexedStep`` remains a runtime-generated wrapper and is +code; it no longer validates an untrusted recipe dictionary. Common definition +fields are dumped once by ``Step.build_step()`` and passed to the existing +constructors. ``IndexedStep`` remains a runtime-generated wrapper and is never added to the authorable model union. Synthetic runtime operations are also constructed directly. For example, @@ -273,8 +266,8 @@ Canonical documentation recipe This documentation-owned fixture demonstrates the version 2 header, two sequences, nested execution, canonical step names, explicit discriminators, indexed input, every input variant, and representative verdict, storage, -image, and passthrough outputs. Tests validate and round-trip it with the -spike. It is not a production bundled recipe. +image, and passthrough outputs. Tests validate and round-trip it with the +production parser. It is not a bundled recipe. .. literalinclude:: _examples/recipe_v2.yml :language: yaml @@ -302,11 +295,9 @@ directory under ``docs/source``. The schema is copied into the HTML output as a download, and the generated RST is included in the toctree. Neither generated file is maintained manually or treated as a committed source artifact. -The documentation environment therefore installs Pydantic. CI and any -documentation build image must install the ``doc`` extra and include the model, -schema generator, and JSON-only renderer sources. This is a build-time -dependency for Phase 5; it does not make Pydantic a production runtime -dependency by itself. +Pydantic is a core production dependency. CI and any documentation build image +must install the ``doc`` extra and include the model, schema generator, and +JSON-only renderer sources. When adding a step or mapping, update its Pydantic model and discriminated union, add an independent round-trip fixture, and run the documentation build. diff --git a/docs/source/yaml_format.rst b/docs/source/yaml_format.rst index e175737..5e6af14 100644 --- a/docs/source/yaml_format.rst +++ b/docs/source/yaml_format.rst @@ -4,12 +4,10 @@ .. important:: - This page documents the recipe syntax currently accepted by the production - runtime (recipe language version 1). The accepted future Pydantic design is - recipe language ``2.0.0``; see :doc:`recipe_language_architecture` for its - architecture and :doc:`_generated/recipe_language_reference` for its - generated syntax reference. Do not use the version 2 example with - production execution until the integration phase is complete. + Production parsing and execution require recipe language ``2.0.0``. See + :doc:`recipe_language_architecture` for its flow and + :doc:`_generated/recipe_language_reference` for the generated syntax + reference. Bundled version 1 examples remain unmigrated and are rejected. .. _yaml_format: diff --git a/pyproject.toml b/pyproject.toml index 237993a..f814094 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ dependencies = [ "pymeasure==0.15.0", "pyserial", "paramiko", + "pydantic>=2.13,<3", "pyvisa" ] @@ -52,7 +53,6 @@ test = [ ] doc = [ "Sphinx", - "pydantic>=2.13,<3", ] dev = [ # Dependencies for development of the project, such as type checkers, diff --git a/spikes/recipe_pydantic/__init__.py b/spikes/recipe_pydantic/__init__.py index a9eb705..026d8a0 100644 --- a/spikes/recipe_pydantic/__init__.py +++ b/spikes/recipe_pydantic/__init__.py @@ -1,36 +1,18 @@ -"""Isolated Pydantic prototype for recipe-language version 2.""" - -from importlib import import_module -from typing import Any - -_PARSER_EXPORTS = { - "Diagnostic", - "ParseResult", - "RecipeParseError", - "SourcePosition", - "SourceSpan", - "dump_recipe", - "parse_recipe_file", - "parse_recipe_text", -} - - -def __getattr__(name: str) -> Any: - """Keep JSON-only submodules importable without initializing Pydantic.""" - if name == "Recipe": - return getattr(import_module(".models", __name__), name) - if name in _PARSER_EXPORTS: - return getattr(import_module(".parser", __name__), name) - raise AttributeError(name) - - -def render_json_schema() -> str: - return import_module(".artifacts", __name__).render_json_schema() - - -def render_reference() -> str: - return import_module(".artifacts", __name__).rendered_artifacts()[1] - +"""Historical spike namespace; production lives in :mod:`pypts`.""" + +from pypts.recipe_artifacts import render_json_schema +from pypts.recipe_language import Recipe +from pypts.recipe_parser import ( + Diagnostic, + ParseResult, + RecipeParseError, + SourcePosition, + SourceSpan, + dump_recipe, + parse_recipe_file, + parse_recipe_text, +) +from pypts.recipe_reference import render_reference __all__ = [ "Diagnostic", diff --git a/spikes/recipe_pydantic/models.py b/spikes/recipe_pydantic/models.py deleted file mode 100644 index 93a5f59..0000000 --- a/spikes/recipe_pydantic/models.py +++ /dev/null @@ -1,386 +0,0 @@ -"""Authoritative Pydantic model for the candidate recipe language. - -Field declarations intentionally own types, defaults, descriptions, examples, -serialization behavior, and JSON Schema. There is no parallel field registry. -""" - -from __future__ import annotations - -from typing import Annotated, Any, Literal, get_args - -from pydantic import BaseModel, ConfigDict, Field, model_validator -from pydantic_core import PydanticCustomError - - -def described(description: str, *, example: Any = None, **kwargs: Any) -> Any: - """Small spelling helper; returned metadata still lives on each Field.""" - examples = None if example is None else [example] - return Field(description=description, examples=examples, **kwargs) - - -class RecipeModel(BaseModel): - """Strict, immutable base for all authorable structures.""" - - model_config = ConfigDict( - extra="forbid", - frozen=True, - strict=True, - populate_by_name=True, - validate_default=True, - ) - - -class DirectInput(RecipeModel): - """Provides a literal value.""" - - type: Literal["direct"] = described("Input source type.", example="direct") - value: Any = described("Literal input value.", example=1) - indexed: bool = described( - "Expand a list into indexed steps.", example=False, default=False, - exclude_if=lambda value: not value, - ) - - # docs:indexed-direct-start - @model_validator(mode="after") - def indexed_values_are_lists(self) -> DirectInput: - if self.indexed and not isinstance(self.value, list): - raise PydanticCustomError( - "invalid_indexed_input", "Indexed direct input value must be a list." - ) - return self - # docs:indexed-direct-end - - -class LocalInput(RecipeModel): - """Reads a sequence-local variable.""" - - type: Literal["local"] = described("Input source type.", example="local") - local_name: str = described("Local variable name.", example="local_value") - - -class GlobalInput(RecipeModel): - """Reads a recipe-global variable.""" - - type: Literal["global"] = described("Input source type.", example="global") - global_name: str = described("Global variable name.", example="global_value") - - -class MethodInput(RecipeModel): - """Resolves a method reference for the step.""" - - type: Literal["method"] = described("Input source type.", example="method") - value: Any = described("Method reference.", example="helper") - - -type InputMapping = Annotated[ - DirectInput | LocalInput | GlobalInput | MethodInput, - Field(discriminator="type"), -] - - -class PassFailOutput(RecipeModel): - """Interprets the output as a pass/fail verdict.""" - - type: Literal["passfail"] = described("Output mapping type.", example="passfail") - - -class EqualsOutput(RecipeModel): - """Passes when the output equals the configured value.""" - - type: Literal["equals"] = described("Output mapping type.", example="equals") - value: Any = described("Expected value.", example=3) - - -class RangeOutput(RecipeModel): - """Passes when the output is within an inclusive range.""" - - type: Literal["range"] = described("Output mapping type.", example="range") - minimum: Any = described("Minimum accepted value.", example=1, alias="min") - maximum: Any = described("Maximum accepted value.", example=4, alias="max") - - -class PassthroughOutput(RecipeModel): - """Uses the nested result without adding a verdict.""" - - type: Literal["passthrough"] = described("Output mapping type.", example="passthrough") - - -class LocalOutput(RecipeModel): - """Stores the output in a sequence-local variable.""" - - type: Literal["local"] = described("Output mapping type.", example="local") - local_name: str = described("Local destination variable.", example="saved") - - -class GlobalOutput(RecipeModel): - """Stores the output in a recipe-global variable.""" - - type: Literal["global"] = described("Output mapping type.", example="global") - global_name: str = described("Global destination variable.", example="saved") - - -class ImageOutput(RecipeModel): - """Publishes an image output for presentation.""" - - type: Literal["image"] = described("Output mapping type.", example="image") - - -type OutputMapping = Annotated[ - PassFailOutput - | EqualsOutput - | RangeOutput - | PassthroughOutput - | LocalOutput - | GlobalOutput - | ImageOutput, - Field(discriminator="type"), -] - - -class InternalSequenceReference(RecipeModel): - """Reference to another sequence in this recipe.""" - - type: Literal["internal"] = described("Reference kind.", example="internal") - name: str = described("Target sequence name.", example="Calibration") - - -class FileDestination(RecipeModel): - """Destination used by a file-loading step.""" - - type: Literal["local", "global"] = described("Variable scope.", example="local") - variable: str = described("Destination variable name.", example="selected_file") - - -class UploadFile(RecipeModel): - """One local-to-remote SSH upload pair.""" - - local: str = described("Local file or package resource.", example="bin/tool") - remote: str = described("Remote destination path.", example="/tmp/tool") - - -class CommonStep(RecipeModel): - """Fields shared by every authorable step.""" - - step_name: str = described("Human-readable step name.", example="Run test") - description: str = described("Purpose of the step.", example="Run a test operation.") - id: str | None = described("Optional stable step identifier.", example="test-1", default=None) - skip: bool = described("Skip execution.", example=False, default=False) - critical: bool = described( - "Stop on error when policy permits continuation.", example=False, default=False - ) - continue_on_error: bool = described("Per-step error policy.", example=False, default=False) - input_mapping: dict[str, InputMapping] = described( - "Named input sources.", example={}, default_factory=dict - ) - output_mapping: dict[str, OutputMapping] = described( - "Named verdicts and destinations.", example={}, default_factory=dict - ) - - -class PythonModuleStep(CommonStep): - """Calls a method or reads/writes an attribute in a Python module.""" - - steptype: Literal["PythonModuleStep"] = described( - "Canonical registered step type.", example="PythonModuleStep" - ) - action_type: Literal["method", "read_attribute", "write_attribute"] = described( - "Operation performed on the Python module.", example="method" - ) - module: str = described("Python module path.", example="tests.py") - method_name: str | None = described("Method name for method actions.", example="run", default=None) - - # docs:method-name-start - @model_validator(mode="after") - def method_actions_have_names(self) -> PythonModuleStep: - if self.action_type == "method" and not self.method_name: - raise PydanticCustomError( - "missing_method_name", "Method actions require method_name." - ) - return self - # docs:method-name-end - - -class SequenceStep(CommonStep): - """Runs another sequence as a step.""" - - steptype: Literal["SequenceStep"] = described( - "Canonical registered step type.", example="SequenceStep" - ) - sequence: InternalSequenceReference = described( - "Internal sequence reference.", example={"type": "internal", "name": "Calibration"} - ) - - -class UserInteractionStep(CommonStep): - """Displays an operator interaction prompt.""" - - steptype: Literal["UserInteractionStep"] = described( - "Canonical registered step type.", example="UserInteractionStep" - ) - - -class WaitStep(CommonStep): - """Waits for a non-negative duration in seconds.""" - - steptype: Literal["WaitStep"] = described("Canonical registered step type.", example="WaitStep") - - # docs:wait-time-start - @model_validator(mode="after") - def has_wait_time(self) -> WaitStep: - if "wait_time" not in self.input_mapping: - raise PydanticCustomError("missing_required_input", "WaitStep requires input 'wait_time'.") - return self - # docs:wait-time-end - - -class UserLoadingStep(CommonStep): - """Prompts the operator to select a file.""" - - steptype: Literal["UserLoadingStep"] = described( - "Canonical registered step type.", example="UserLoadingStep" - ) - file_save_location: FileDestination | None = described( - "Local or global destination for the selected file.", - example={"type": "local", "variable": "selected_file"}, - default=None, - ) - - -class UserRunMethodStep(CommonStep): - """Optionally runs a Python method after an operator response.""" - - steptype: Literal["UserRunMethodStep"] = described( - "Canonical registered step type.", example="UserRunMethodStep" - ) - trigger_response: str | list[Any] | dict[str, Any] | None = described( - "Operator response that triggers execution.", example="run", default=None - ) - action_type: str | None = described("Optional Python action type.", example="method", default=None) - module: str | None = described("Optional Python module path.", example="tests.py", default=None) - method_name: str | None = described("Optional Python method name.", example="run", default=None) - - -class UserWriteStep(CommonStep): - """Writes an operator-provided value to a configured destination.""" - - steptype: Literal["UserWriteStep"] = described( - "Canonical registered step type.", example="UserWriteStep" - ) - - -class SerialNumberStep(CommonStep): - """Captures the device serial number.""" - - steptype: Literal["SerialNumberStep"] = described( - "Canonical registered step type.", example="SerialNumberStep" - ) - - -class SSHConnectStep(CommonStep): - """Opens the SSH client stored in recipe globals.""" - - steptype: Literal["SSHConnectStep"] = described( - "Canonical registered step type.", example="SSHConnectStep" - ) - - -class SSHCloseStep(CommonStep): - """Closes the SSH client stored in recipe globals.""" - - steptype: Literal["SSHCloseStep"] = described( - "Canonical registered step type.", example="SSHCloseStep" - ) - - -class SSHUploadStep(CommonStep): - """Uploads files through an SSH connection.""" - - steptype: Literal["SSHUploadStep"] = described( - "Canonical registered step type.", example="SSHUploadStep" - ) - files: list[UploadFile] = described( - "Local and remote file pairs to upload.", - example=[{"local": "bin/tool", "remote": "/tmp/tool"}], - ) - permissions: int | str | None = described( - "Optional remote permissions.", example="0755", default=None - ) - skip_if_sha256_match: bool = described( - "Skip files whose remote checksum matches.", example=False, default=False - ) - local_package: str | None = described( - "Optional package containing local resources.", example="my_package", default=None - ) - - -type Step = Annotated[ - PythonModuleStep - | SequenceStep - | UserInteractionStep - | WaitStep - | UserLoadingStep - | UserRunMethodStep - | UserWriteStep - | SerialNumberStep - | SSHConnectStep - | SSHCloseStep - | SSHUploadStep, - Field(discriminator="steptype"), -] - - -class RecipeHeader(RecipeModel): - """The first YAML document, identifying a recipe and its entry sequence.""" - - name: str = described("Human-readable recipe name.", example="Hardware acceptance") - version: str = described("Version of this recipe.", example="1.0") - recipe_version: Literal["2.0.0"] = described( - "Version of the recipe language contract.", example="2.0.0" - ) - description: str = described("Purpose of the recipe.", example="Acceptance tests.") - main_sequence: str = described("Sequence where execution begins.", example="Main") - globals: dict[str, Any] = described("Recipe-wide variables.", example={}) - continue_on_error: bool | None = described( - "Recipe-wide error policy.", example=False, default=None - ) - report: Literal["overwrite", "append"] = described( - "Report file mode.", example="overwrite", default="overwrite" - ) - report_name_include_serial: bool = described( - "Include the serial number in the report name.", example=False, default=False - ) - test_package: str | None = described( - "Package containing recipe test modules.", example="acceptance", default=None - ) - - -class Sequence(RecipeModel): - """One named executable sequence document.""" - - sequence_name: str = described("Unique sequence name.", example="Main") - description: str = described("Purpose of the sequence.", example="Main sequence.") - parameters: dict[str, Any] = described("Reserved sequence input metadata.", example={}) - outputs: dict[str, Any] = described("Reserved sequence output metadata.", example={}) - locals: dict[str, Any] = described("Variables local to the sequence.", example={}) - setup_steps: list[Step] = described("Steps run before the main steps.", example=[]) - steps: list[Step] = described("Ordered main steps.", example=[]) - teardown_steps: list[Step] = described("Steps run during teardown.", example=[]) - - -class Recipe(RecipeModel): - """Aggregate typed recipe used by tooling and JSON Schema consumers.""" - - header: RecipeHeader = described("Recipe header document.") - sequences: list[Sequence] = described("Sequence documents.", min_length=1) - - -def _union_models(annotation: Any) -> tuple[type[RecipeModel], ...]: - """Expose union members for generators without a second registry.""" - annotation = getattr(annotation, "__value__", annotation) - annotated_union = get_args(annotation)[0] - return get_args(annotated_union) - - -STEP_MODELS = _union_models(Step) -INPUT_MODELS = _union_models(InputMapping) -OUTPUT_MODELS = _union_models(OutputMapping) diff --git a/spikes/recipe_pydantic/parser.py b/spikes/recipe_pydantic/parser.py deleted file mode 100644 index 729dd78..0000000 --- a/spikes/recipe_pydantic/parser.py +++ /dev/null @@ -1,542 +0,0 @@ -"""Safe YAML adapter and semantic validation for the Pydantic v2 spike.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import yaml -from pydantic import ValidationError - -from .models import ( - DirectInput, - EqualsOutput, - PassFailOutput, - PassthroughOutput, - RangeOutput, - Recipe, - RecipeHeader, - Sequence, - SequenceStep, -) - -type RecipePath = tuple[str | int, ...] - - -@dataclass(frozen=True) -class SourcePosition: - """A one-based source position with a zero-based character offset.""" - - line: int - column: int - offset: int - - -@dataclass(frozen=True) -class SourceSpan: - """Half-open source range.""" - - start: SourcePosition - end: SourcePosition - - -@dataclass(frozen=True) -class Diagnostic: - """Source-aware recipe finding, compatible with the PyPTS envelope.""" - - code: str - message: str - path: RecipePath = () - severity: str = "error" - source_name: str | None = None - span: SourceSpan | None = None - - -class RecipeParseError(ValueError): - """Raised when a caller requires a recipe from an unsuccessful parse.""" - - def __init__(self, diagnostics: tuple[Diagnostic, ...]): - self.diagnostics = diagnostics - errors = sum(item.severity == "error" for item in diagnostics) - super().__init__(f"Recipe parsing failed with {errors} error(s).") - - -@dataclass(frozen=True) -class ParseResult: - recipe: Recipe | None - diagnostics: tuple[Diagnostic, ...] = () - - @property - def errors(self) -> tuple[Diagnostic, ...]: - return tuple(item for item in self.diagnostics if item.severity == "error") - - @property - def warnings(self) -> tuple[Diagnostic, ...]: - return tuple(item for item in self.diagnostics if item.severity == "warning") - - @property - def is_valid(self) -> bool: - return self.recipe is not None and not self.errors - - def require_recipe(self) -> Recipe: - if not self.is_valid: - raise RecipeParseError(self.diagnostics) - assert self.recipe is not None - return self.recipe - - -def _position(mark: yaml.error.Mark) -> SourcePosition: - return SourcePosition(mark.line + 1, mark.column + 1, mark.index) - - -def _span(node: yaml.Node) -> SourceSpan: - return SourceSpan(_position(node.start_mark), _position(node.end_mark)) - - -def _mark_span(mark: yaml.error.Mark | None) -> SourceSpan | None: - if mark is None: - return None - position = _position(mark) - return SourceSpan(position, position) - - -def _nearest_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: - candidate = path - while candidate: - if candidate in spans: - return spans[candidate] - candidate = candidate[:-1] - return spans.get(()) - - -def _diagnostic( - code: str, - message: str, - path: RecipePath, - source_name: str, - spans: Mapping[RecipePath, SourceSpan], -) -> Diagnostic: - return Diagnostic(code, message, path, source_name=source_name, span=_nearest_span(path, spans)) - - -def _index_nodes( - node: yaml.Node, - path: RecipePath, - spans: dict[RecipePath, SourceSpan], - diagnostics: list[Diagnostic], - source_name: str, - active: set[int], -) -> None: - spans[path] = _span(node) - identity = id(node) - if identity in active: - diagnostics.append(Diagnostic( - "recursive-alias", - "Recursive YAML aliases are not supported.", - path, - source_name=source_name, - span=_span(node), - )) - return - active.add(identity) - try: - if isinstance(node, yaml.MappingNode): - seen: set[tuple[str, str]] = set() - for key_node, value_node in node.value: - key_identity = (key_node.tag, repr(key_node.value)) - key: str | int = key_node.value if isinstance(key_node, yaml.ScalarNode) else repr(key_node.value) - child = path + (key,) - if key_identity in seen: - diagnostics.append(Diagnostic( - "duplicate-key", - f"Duplicate YAML key '{key}'.", - child, - source_name=source_name, - span=_span(key_node), - )) - seen.add(key_identity) - _index_nodes(value_node, child, spans, diagnostics, source_name, active) - elif isinstance(node, yaml.SequenceNode): - for index, child_node in enumerate(node.value): - _index_nodes( - child_node, path + (index,), spans, diagnostics, source_name, active - ) - finally: - active.remove(identity) - - -_MODEL_TAGS = { - "direct", "local", "global", "method", "passfail", "equals", "range", - "passthrough", "image", "PythonModuleStep", "SequenceStep", - "UserInteractionStep", "WaitStep", "UserLoadingStep", "UserRunMethodStep", - "UserWriteStep", "SerialNumberStep", "SSHConnectStep", "SSHCloseStep", - "SSHUploadStep", -} -_CANONICAL_STEPS = {tag for tag in _MODEL_TAGS if tag.endswith("Step")} - - -def _clean_location(location: tuple[Any, ...]) -> RecipePath: - cleaned: list[str | int] = [] - for index, item in enumerate(location): - is_step_tag = ( - item in _CANONICAL_STEPS - and index >= 2 - and location[index - 2] in {"setup_steps", "steps", "teardown_steps"} - and isinstance(location[index - 1], int) - ) - is_mapping_tag = ( - item in _MODEL_TAGS - and index >= 2 - and location[index - 2] in {"input_mapping", "output_mapping"} - ) - if not is_step_tag and not is_mapping_tag: - cleaned.append(item) - return tuple(cleaned) - - -def _pydantic_diagnostic( - error: dict[str, Any], - prefix: RecipePath, - source_name: str, - spans: Mapping[RecipePath, SourceSpan], -) -> Diagnostic: - location = prefix + _clean_location(tuple(error.get("loc", ()))) - kind = error["type"] - context = error.get("ctx") or {} - input_value = error.get("input") - code = "invalid-field" - message = error["msg"] - - if kind == "missing": - code = "missing-field" - elif kind == "extra_forbidden": - field = location[-1] if location else "field" - if field == "serial_number": - code = "removed-sequence-field" - message = "Sequence field 'serial_number' was removed in recipe language 2.0.0." - else: - code = "unknown-field" - message = f"Unknown field '{field}'." - elif kind == "union_tag_not_found": - discriminator = str(context.get("discriminator", "")) - if "steptype" in discriminator: - code = "missing-step-type" - location += ("steptype",) - message = "Step requires canonical 'steptype'." - elif "output_mapping" in location: - code = "missing-output-type" - location += ("type",) - message = "Output mapping requires an explicit 'type'." - else: - code = "missing-input-type" - location += ("type",) - message = "Mapping requires an explicit 'type' in recipe language 2.0.0." - elif kind == "union_tag_invalid": - discriminator = str(context.get("discriminator", "")) - tag = context.get("tag") - if "steptype" in discriminator: - location += ("steptype",) - canonical = next( - (candidate for candidate in _CANONICAL_STEPS if candidate.casefold() == str(tag).casefold()), - None, - ) - if canonical: - code = "noncanonical-step-type" - message = f"Use canonical step type '{canonical}' instead of '{tag}'." - else: - code = "unknown-step-type" - message = f"Unknown step type '{tag}'." - else: - location += ("type",) - if "output_mapping" in location: - code = "unknown-output-type" - message = f"Unknown output mapping type '{tag}'." - else: - code = "unknown-input-type" - message = f"Unknown input mapping type '{tag}'." - elif kind == "literal_error" and location[-1:] == ("recipe_version",): - code = "unsupported-recipe-version" - message = f"Recipe language version {input_value!r} is unsupported; expected '2.0.0'." - elif kind == "literal_error": - code = "invalid-field-value" - elif kind in {"bool_type", "string_type", "int_type", "list_type", "dict_type", "model_type"}: - code = "invalid-field-type" - elif kind == "invalid_indexed_input": - code = "invalid-indexed-input" - elif kind == "missing_method_name": - code = "missing-method-name" - location += ("method_name",) - elif kind == "missing_required_input": - code = "missing-required-input" - location += ("input_mapping", "wait_time") - elif kind == "too_short" and prefix == () and location[-1:] == ("sequences",): - code = "missing-sequence" - - return _diagnostic(code, message, location, source_name, spans) - - -def _validation_diagnostics( - error: ValidationError, - prefix: RecipePath, - source_name: str, - spans: Mapping[RecipePath, SourceSpan], -) -> list[Diagnostic]: - return [ - _pydantic_diagnostic(item, prefix, source_name, spans) - for item in error.errors(include_url=False) - ] - - -def _all_steps(sequence: Sequence): - for section_name in ("setup_steps", "steps", "teardown_steps"): - for index, step in enumerate(getattr(sequence, section_name)): - yield section_name, index, step - - -def _semantic_diagnostics( - header: RecipeHeader | None, - sequences: list[tuple[int, Sequence]], - source_name: str, - spans: Mapping[RecipePath, SourceSpan], - *, - complete_sequences: bool = True, -) -> list[Diagnostic]: - """Rules that cannot be expressed by one structural Pydantic model.""" - diagnostics: list[Diagnostic] = [] - # docs:sequence-semantics-start - by_name: dict[str, tuple[int, Sequence]] = {} - for document_index, sequence in sequences: - path = (document_index, "sequence_name") - if sequence.sequence_name in by_name: - diagnostics.append(_diagnostic( - "duplicate-sequence", - f"Duplicate sequence '{sequence.sequence_name}'.", - path, - source_name, - spans, - )) - else: - by_name[sequence.sequence_name] = (document_index, sequence) - - if complete_sequences and header is not None and header.main_sequence not in by_name: - diagnostics.append(_diagnostic( - "unknown-main-sequence", - f"Main sequence '{header.main_sequence}' does not exist.", - (0, "main_sequence"), - source_name, - spans, - )) - # docs:sequence-semantics-end - - verdict_types = (PassFailOutput, EqualsOutput, RangeOutput, PassthroughOutput) - for document_index, sequence in sequences: - flattened = list(_all_steps(sequence)) - for section, index, step in flattened: - step_path = (document_index, section, index) - # docs:nested-reference-start - if isinstance(step, SequenceStep) and step.sequence.name not in by_name: - diagnostics.append(_diagnostic( - "unknown-sequence-reference", - f"Sequence '{sequence.sequence_name}' references unknown sequence " - f"'{step.sequence.name}'.", - step_path + ("sequence", "name"), - source_name, - spans, - )) - # docs:nested-reference-end - - # docs:mapping-semantics-start - indexed_lengths = [ - len(value.value) - for value in step.input_mapping.values() - if isinstance(value, DirectInput) and value.indexed - ] - if len(set(indexed_lengths)) > 1: - diagnostics.append(_diagnostic( - "unequal-indexed-inputs", - "Indexed input lists must have equal lengths.", - step_path + ("input_mapping",), - source_name, - spans, - )) - - verdicts = [ - value for value in step.output_mapping.values() if isinstance(value, verdict_types) - ] - if any(isinstance(value, PassthroughOutput) for value in verdicts) and len(verdicts) != 1: - diagnostics.append(_diagnostic( - "mixed-passthrough", - "'passthrough' must be the sole verdict mapping.", - step_path + ("output_mapping",), - source_name, - spans, - )) - # docs:mapping-semantics-end - - # docs:ssh-semantics-start - ssh_steps = [item for item in flattened if item[2].steptype.startswith("SSH")] - if ssh_steps and header is not None: - for required in ("ssh_client", "host", "user", "port"): - if required not in header.globals: - diagnostics.append(_diagnostic( - "missing-ssh-global", - f"SSH step requires global '{required}'.", - (0, "globals", required), - source_name, - spans, - )) - if "password" not in header.globals and "private_key" not in header.globals: - diagnostics.append(_diagnostic( - "missing-ssh-credential", - "SSH steps require password or private_key global.", - (0, "globals"), - source_name, - spans, - )) - - connected = False - unclosed_connect: tuple[str, int] | None = None - for section, index, step in flattened: - if step.steptype == "SSHConnectStep": - connected = True - unclosed_connect = (section, index) - elif step.steptype == "SSHUploadStep" and not connected: - diagnostics.append(_diagnostic( - "missing-ssh-connect", - f"Sequence '{sequence.sequence_name}' uploads before an SSH connection.", - (document_index, section, index), - source_name, - spans, - )) - elif step.steptype == "SSHCloseStep": - connected = False - unclosed_connect = None - if unclosed_connect is not None: - diagnostics.append(_diagnostic( - "missing-ssh-close", - f"Sequence '{sequence.sequence_name}' opens SSH without a later close.", - (document_index, "teardown_steps"), - source_name, - spans, - )) - # docs:ssh-semantics-end - return diagnostics - - -def parse_recipe_text(text: str, source_name: str = "") -> ParseResult: - """Parse candidate recipe-language 2 YAML without runtime or GUI imports.""" - if not isinstance(text, str): - return ParseResult(None, (Diagnostic( - "invalid-source", "Recipe source must be text.", source_name=source_name - ),)) - if not text.strip(): - return ParseResult(None, (Diagnostic( - "empty-recipe", "A recipe requires a header and at least one sequence.", - source_name=source_name, - ),)) - - try: - nodes = list(yaml.compose_all(text, Loader=yaml.SafeLoader)) - except yaml.YAMLError as error: - mark = getattr(error, "problem_mark", None) - return ParseResult(None, (Diagnostic( - "yaml-syntax-error", str(error), source_name=source_name, span=_mark_span(mark) - ),)) - - spans: dict[RecipePath, SourceSpan] = {} - diagnostics: list[Diagnostic] = [] - for index, node in enumerate(nodes): - if node is not None: - _index_nodes(node, (index,), spans, diagnostics, source_name, set()) - - try: - documents = list(yaml.safe_load_all(text)) - except yaml.YAMLError as error: - mark = getattr(error, "problem_mark", None) - code = "unsafe-yaml" if isinstance(error, yaml.constructor.ConstructorError) else "yaml-construction-error" - diagnostics.append(Diagnostic(code, str(error), source_name=source_name, span=_mark_span(mark))) - return ParseResult(None, tuple(diagnostics)) - - if not documents or all(document is None for document in documents): - diagnostics.append(Diagnostic( - "empty-recipe", "A recipe requires a header and at least one sequence.", - source_name=source_name, - )) - return ParseResult(None, tuple(diagnostics)) - - header: RecipeHeader | None = None - semantic_header: RecipeHeader | None = None - raw_header = documents[0] - try: - header = RecipeHeader.model_validate(raw_header) - semantic_header = header - except ValidationError as error: - diagnostics.extend(_validation_diagnostics(error, (0,), source_name, spans)) - if isinstance(raw_header, dict) and raw_header.get("recipe_version") != "2.0.0": - candidate = dict(raw_header) - candidate["recipe_version"] = "2.0.0" - try: - semantic_header = RecipeHeader.model_validate(candidate) - except ValidationError: - pass - - sequences: list[tuple[int, Sequence]] = [] - complete_sequences = True - for index, document in enumerate(documents[1:], start=1): - try: - sequences.append((index, Sequence.model_validate(document))) - except ValidationError as error: - complete_sequences = False - diagnostics.extend(_validation_diagnostics(error, (index,), source_name, spans)) - - if len(documents) == 1: - diagnostics.append(_diagnostic( - "missing-sequence", "A recipe requires at least one sequence.", (0,), source_name, spans - )) - - diagnostics.extend(_semantic_diagnostics( - semantic_header, - sequences, - source_name, - spans, - complete_sequences=complete_sequences, - )) - if diagnostics: - return ParseResult(None, tuple(diagnostics)) - assert header is not None - recipe = Recipe(header=header, sequences=[sequence for _, sequence in sequences]) - return ParseResult(recipe) - - -def parse_recipe_file(path: str | Path, encoding: str = "utf-8") -> ParseResult: - """Read and parse a candidate recipe file.""" - source_path = Path(path) - try: - text = source_path.read_text(encoding=encoding) - except (OSError, UnicodeError) as error: - return ParseResult(None, (Diagnostic( - "file-read-error", f"Could not read recipe: {error}", source_name=str(source_path) - ),)) - return parse_recipe_text(text, str(source_path)) - - -def dump_recipe(recipe: Recipe) -> str: - """Serialize a typed recipe as canonical multi-document YAML.""" - if not isinstance(recipe, Recipe): - raise TypeError("dump_recipe expects a Recipe") - documents = [ - recipe.header.model_dump(mode="python", by_alias=True, exclude_none=True), - *[ - sequence.model_dump(mode="python", by_alias=True, exclude_none=True) - for sequence in recipe.sequences - ], - ] - return yaml.safe_dump_all( - documents, - explicit_start=True, - sort_keys=False, - default_flow_style=False, - allow_unicode=True, - ) diff --git a/spikes/recipe_pydantic/reference.py b/spikes/recipe_pydantic/reference.py deleted file mode 100644 index 028645d..0000000 --- a/spikes/recipe_pydantic/reference.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Render the Sphinx recipe reference from generated JSON Schema only.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - - -def load_schema(path: str | Path) -> dict[str, Any]: - """Load and minimally verify an aggregate recipe JSON Schema.""" - source = Path(path) - schema = json.loads(source.read_text(encoding="utf-8")) - if not isinstance(schema, dict) or not isinstance(schema.get("$defs"), dict): - raise TypeError(f"Recipe schema has no $defs object: {source}") - return schema - - -def _reference_name(reference: str) -> str: - prefix = "#/$defs/" - if not reference.startswith(prefix): - raise ValueError(f"Unsupported external JSON Schema reference: {reference}") - return reference.removeprefix(prefix) - - -def _resolve(value: dict[str, Any], definitions: dict[str, Any]) -> dict[str, Any]: - if "$ref" not in value: - return value - return definitions[_reference_name(value["$ref"])] - - -def _type_name(value: dict[str, Any]) -> str: - if "$ref" in value: - return _reference_name(value["$ref"]) - if "const" in value: - return repr(value["const"]) - if "enum" in value: - return " | ".join(repr(item) for item in value["enum"]) - if "anyOf" in value: - return " | ".join(_type_name(item) for item in value["anyOf"]) - kind = value.get("type") - if kind == "array": - return f"list[{_type_name(value.get('items', {}))}]" - if kind == "object": - additional = value.get("additionalProperties") - if isinstance(additional, dict): - return f"dict[str, {_type_name(additional)}]" - return "object" - return { - "boolean": "bool", - "integer": "int", - "number": "number", - "null": "None", - "string": "str", - }.get(kind, "any") - - -def _literal(value: Any) -> str: - return json.dumps(value, ensure_ascii=False, sort_keys=True) - - -def _field_table( - definition: dict[str, Any], - *, - include: set[str] | None = None, - exclude: set[str] | None = None, -) -> list[str]: - properties = definition.get("properties", {}) - required = set(definition.get("required", [])) - names = [ - name for name in properties - if (include is None or name in include) and (exclude is None or name not in exclude) - ] - if not names: - return ["This variant adds no fields.", ""] - # REUSE-IgnoreStart - lines = [ - ".. list-table:: Fields", - " :header-rows: 1", - " :widths: 18 19 25 38", - "", - " * - Field", - " - Type", - " - Requirement", - " - Description and example", - ] - for name in names: - field = properties[name] - requirement = "required" if name in required else "optional" - if "default" in field: - requirement += f"; default ``{_literal(field['default'])}``" - details = field.get("description", "") - examples = field.get("examples", []) - if examples: - details += f" Example: ``{_literal(examples[0])}``." - lines.extend(( - f" * - ``{name}``", - f" - ``{_type_name(field)}``", - f" - {requirement}", - f" - {details}", - )) - lines.append("") - return lines - - -def _model_section( - definition_name: str, - definitions: dict[str, Any], - anchor: str, - *, - include: set[str] | None = None, - exclude: set[str] | None = None, -) -> list[str]: - definition = definitions[definition_name] - title = definition.get("title", definition_name) - lines = [f".. _{anchor}:", "", title, "~" * len(title), ""] - if definition.get("description"): - lines.extend((definition["description"], "")) - lines.extend(_field_table(definition, include=include, exclude=exclude)) - return lines - - -def _discriminator_mapping( - definitions: dict[str, Any], name: str -) -> dict[str, str]: - definition = definitions[name] - mapping = definition.get("discriminator", {}).get("mapping") - if not isinstance(mapping, dict) or not mapping: - raise ValueError(f"$defs.{name} has no discriminator mapping") - return {key: _reference_name(reference) for key, reference in mapping.items()} - - -def _common_step_fields( - definitions: dict[str, Any], step_names: list[str] -) -> set[str]: - property_sets = [set(definitions[name].get("properties", {})) for name in step_names] - common = set.intersection(*property_sets) - result: set[str] = set() - for field_name in common: - values = [definitions[name]["properties"][field_name] for name in step_names] - if all(value == values[0] for value in values[1:]): - result.add(field_name) - return result - - -def render_reference(schema: dict[str, Any]) -> str: - """Render deterministic RST using only a parsed JSON Schema document.""" - definitions = schema["$defs"] - steps = _discriminator_mapping(definitions, "Step") - inputs = _discriminator_mapping(definitions, "InputMapping") - outputs = _discriminator_mapping(definitions, "OutputMapping") - common_fields = _common_step_fields(definitions, list(steps.values())) - - lines = [ - ".. SPDX-FileCopyrightText: 2026 CERN ", - "..", - ".. SPDX-License-Identifier: CC-BY-SA-4.0", - "..", - ".. Generated from recipe_language.schema.json. Do not edit manually.", - "", - "Recipe Language 2.0 Reference", - "=============================", - "", - "This page is generated from the current build's aggregate JSON Schema. It", - "describes", - "the accepted future recipe language model; production execution still uses", - "the version 1 language until Phase 6 integration is complete.", - "", - ":download:`Download the JSON Schema `.", - "", - "See :doc:`/recipe_language_architecture` for parsing, semantic rules,", - "documentation maintenance, and the planned YamVIEW and sequencer flows.", - "", - "Documents", - "---------", - "", - ] - # REUSE-IgnoreEnd - lines.extend(_model_section("RecipeHeader", definitions, "recipe-v2-header")) - lines.extend(_model_section("Sequence", definitions, "recipe-v2-sequence")) - - lines.extend(("Nested structures", "-----------------", "")) - for name in ("InternalSequenceReference", "FileDestination", "UploadFile"): - lines.extend(_model_section(name, definitions, f"recipe-v2-structure-{name.lower()}")) - - lines.extend(("Common step fields", "------------------", "")) - representative = next(iter(steps.values())) - lines.extend(_field_table(definitions[representative], include=common_fields)) - - lines.extend(("Authorable steps", "----------------", "")) - for discriminator, definition_name in steps.items(): - lines.extend(_model_section( - definition_name, - definitions, - f"recipe-v2-step-{discriminator.lower()}", - exclude=common_fields, - )) - - lines.extend(("Input mappings", "--------------", "")) - for discriminator, definition_name in inputs.items(): - lines.extend(_model_section( - definition_name, definitions, f"recipe-v2-input-{discriminator}" - )) - - lines.extend(("Output mappings", "---------------", "")) - for discriminator, definition_name in outputs.items(): - lines.extend(_model_section( - definition_name, definitions, f"recipe-v2-output-{discriminator}" - )) - return "\n".join(lines).rstrip() + "\n" - - -def render_reference_file(path: str | Path) -> str: - return render_reference(load_schema(path)) diff --git a/spikes/recipe_pydantic/test_recipe_pydantic.py b/spikes/recipe_pydantic/test_recipe_pydantic.py deleted file mode 100644 index ff4058d..0000000 --- a/spikes/recipe_pydantic/test_recipe_pydantic.py +++ /dev/null @@ -1,410 +0,0 @@ -# SPDX-FileCopyrightText: 2026 CERN -# SPDX-License-Identifier: LGPL-2.1-or-later -"""Executable evaluation suite for the isolated Pydantic recipe spike.""" - -from __future__ import annotations - -import ast -import json -from pathlib import Path - -import pytest -import yaml -from pydantic import ValidationError - -from pypts.recipe_parser import dump_recipe as dump_v1 -from pypts.recipe_parser import parse_recipe_file as parse_v1 - -from .artifacts import ( - DEFAULT_REFERENCE_PATH, - DEFAULT_SCHEMA_PATH, - render_json_schema, -) -from .models import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS -from .parser import RecipeParseError, dump_recipe, parse_recipe_file, parse_recipe_text -from .reference import render_reference - -ROOT = Path(__file__).parents[2] -RECIPES = ROOT / "src" / "pypts" / "recipes" - - -def header(**updates): - value = { - "name": "Pydantic spike", - "version": "1.0", - "recipe_version": "2.0.0", - "description": "Candidate recipe.", - "main_sequence": "Main", - "globals": {}, - } - value.update(updates) - return value - - -def sequence(steps=None, **updates): - value = { - "sequence_name": "Main", - "description": "Main sequence.", - "parameters": {}, - "outputs": {}, - "locals": {}, - "setup_steps": [], - "steps": steps or [], - "teardown_steps": [], - } - value.update(updates) - return value - - -def source(*documents): - return yaml.safe_dump_all(documents, explicit_start=True, sort_keys=False) - - -def common(kind, **updates): - value = { - "steptype": kind, - "step_name": kind, - "description": f"Exercise {kind}.", - "input_mapping": {}, - "output_mapping": {}, - } - value.update(updates) - return value - - -STEP_EXAMPLES = { - "PythonModuleStep": common( - "PythonModuleStep", action_type="method", module="tests.py", method_name="run" - ), - "SequenceStep": common( - "SequenceStep", sequence={"type": "internal", "name": "Target"} - ), - "UserInteractionStep": common("UserInteractionStep"), - "WaitStep": common( - "WaitStep", input_mapping={"wait_time": {"type": "direct", "value": 0}} - ), - "UserLoadingStep": common( - "UserLoadingStep", - file_save_location={"type": "local", "variable": "selected"}, - ), - "UserRunMethodStep": common( - "UserRunMethodStep", - trigger_response="run", - action_type="method", - module="tests.py", - method_name="run", - ), - "UserWriteStep": common("UserWriteStep"), - "SerialNumberStep": common("SerialNumberStep"), - "SSHConnectStep": common("SSHConnectStep"), - "SSHCloseStep": common("SSHCloseStep"), - "SSHUploadStep": common( - "SSHUploadStep", - files=[{"local": "bin/tool", "remote": "/tmp/tool"}], - permissions="0755", - skip_if_sha256_match=True, - local_package="fixtures", - ), -} - - -def recipe_for_step(step): - globals_value = { - "ssh_client": None, - "host": "target", - "user": "root", - "port": 22, - "password": "secret", - } - main = sequence([step]) - documents = [header(globals=globals_value), main] - if step["steptype"] == "SequenceStep": - documents.append(sequence(sequence_name="Target")) - elif step["steptype"] == "SSHUploadStep": - main["setup_steps"] = [common("SSHConnectStep")] - main["teardown_steps"] = [common("SSHCloseStep")] - elif step["steptype"] == "SSHConnectStep": - main["teardown_steps"] = [common("SSHCloseStep")] - return source(*documents) - - -@pytest.mark.parametrize("model", STEP_MODELS, ids=lambda model: model.__name__) -def test_every_step_validates_serializes_and_reparses(model): - first = parse_recipe_text(recipe_for_step(STEP_EXAMPLES[model.__name__])) - assert first.is_valid, first.errors - second = parse_recipe_text(dump_recipe(first.require_recipe())) - assert second.is_valid, second.errors - assert second.recipe == first.recipe - - -INPUT_EXAMPLES = { - "DirectInput": {"type": "direct", "value": [1, 2], "indexed": True}, - "LocalInput": {"type": "local", "local_name": "local_value"}, - "GlobalInput": {"type": "global", "global_name": "global_value"}, - "MethodInput": {"type": "method", "value": "helper"}, -} - - -@pytest.mark.parametrize("model", INPUT_MODELS, ids=lambda model: model.__name__) -def test_every_input_mapping_validates_serializes_and_reparses(model): - step = STEP_EXAMPLES["PythonModuleStep"] | { - "input_mapping": {"example": INPUT_EXAMPLES[model.__name__]} - } - first = parse_recipe_text(recipe_for_step(step)) - assert isinstance(first.require_recipe().sequences[0].steps[0].input_mapping["example"], model) - assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe - - -OUTPUT_EXAMPLES = { - "PassFailOutput": {"type": "passfail"}, - "EqualsOutput": {"type": "equals", "value": 3}, - "RangeOutput": {"type": "range", "min": 1, "max": 4}, - "PassthroughOutput": {"type": "passthrough"}, - "LocalOutput": {"type": "local", "local_name": "saved"}, - "GlobalOutput": {"type": "global", "global_name": "saved"}, - "ImageOutput": {"type": "image"}, -} - - -@pytest.mark.parametrize("model", OUTPUT_MODELS, ids=lambda model: model.__name__) -def test_every_output_mapping_validates_serializes_and_reparses(model): - step = STEP_EXAMPLES["PythonModuleStep"] | { - "output_mapping": {"example": OUTPUT_EXAMPLES[model.__name__]} - } - first = parse_recipe_text(recipe_for_step(step)) - assert isinstance(first.require_recipe().sequences[0].steps[0].output_mapping["example"], model) - assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe - - -def test_defaults_are_typed_dumped_and_models_are_frozen(): - recipe = parse_recipe_text(recipe_for_step(STEP_EXAMPLES["UserInteractionStep"])).require_recipe() - step = recipe.sequences[0].steps[0] - assert step.skip is step.critical is step.continue_on_error is False - assert recipe.header.report == "overwrite" - dumped = dump_recipe(recipe) - assert "report: overwrite" in dumped and "skip: false" in dumped - with pytest.raises(ValidationError): - step.skip = True - - -def test_strict_types_unknown_fields_and_structural_rules_are_rejected(): - bad = STEP_EXAMPLES["PythonModuleStep"] | { - "skip": 0, - "surprise": True, - "method_name": None, - } - codes = {item.code for item in parse_recipe_text(recipe_for_step(bad)).errors} - assert {"invalid-field-type", "unknown-field"} <= codes - - missing_method = STEP_EXAMPLES["PythonModuleStep"] | {"method_name": None} - assert "missing-method-name" in { - item.code for item in parse_recipe_text(recipe_for_step(missing_method)).errors - } - - wait = common("WaitStep") - assert "missing-required-input" in { - item.code for item in parse_recipe_text(recipe_for_step(wait)).errors - } - - nested = STEP_EXAMPLES["PythonModuleStep"] | { - "input_mapping": {"local": {"type": "local", "local_name": 1}} - } - finding = next( - item for item in parse_recipe_text(recipe_for_step(nested)).errors - if item.code == "invalid-field-type" - ) - assert finding.path[-2:] == ("local", "local_name") - - -def test_discriminators_are_explicit_and_canonical(): - lowercase = common( - "waitstep", input_mapping={"wait_time": {"type": "direct", "value": 1}} - ) - omitted_type = STEP_EXAMPLES["WaitStep"] | { - "input_mapping": {"wait_time": {"value": 1}} - } - unknown = common("InventedStep") - missing_output_type = STEP_EXAMPLES["PythonModuleStep"] | { - "output_mapping": {"result": {"value": 1}} - } - assert {item.code for item in parse_recipe_text(recipe_for_step(lowercase)).errors} == { - "noncanonical-step-type" - } - assert {item.code for item in parse_recipe_text(recipe_for_step(omitted_type)).errors} == { - "missing-input-type" - } - assert {item.code for item in parse_recipe_text(recipe_for_step(unknown)).errors} == { - "unknown-step-type" - } - assert { - item.code for item in parse_recipe_text(recipe_for_step(missing_output_type)).errors - } == {"missing-output-type"} - - -def test_v1_migration_errors_are_aggregated_across_documents(): - legacy = header(recipe_version="1.0.0") - first = sequence( - [common("waitstep", input_mapping={"wait_time": {"value": 1}})], - serial_number=12, - ) - second = sequence( - [STEP_EXAMPLES["WaitStep"] | { - "input_mapping": {"wait_time": {"value": 1}} - }], - sequence_name="Other", - serial_number="old", - ) - result = parse_recipe_text(source(legacy, first, second), "legacy.yml") - codes = [item.code for item in result.errors] - assert codes.count("removed-sequence-field") == 2 - assert {"unsupported-recipe-version", "noncanonical-step-type", "missing-input-type"} <= set(codes) - assert all(item.source_name == "legacy.yml" and item.span is not None for item in result.errors) - - -def test_source_spans_point_to_fields_and_nearest_parent(): - text = source(header(main_sequence="Missing"), sequence()) - result = parse_recipe_text(text, "broken.yml") - finding = next(item for item in result.errors if item.code == "unknown-main-sequence") - expected = next( - index for index, line in enumerate(text.splitlines(), start=1) - if line.startswith("main_sequence:") - ) - assert finding.source_name == "broken.yml" - assert finding.span is not None and finding.span.start.line == expected - - missing = source(header(), sequence()).replace("description: Main sequence.\n", "") - finding = next( - item for item in parse_recipe_text(missing).errors - if item.code == "missing-field" and item.path[-1] == "description" - ) - assert finding.span is not None and finding.span.start.line > 1 - - -def test_yaml_failures_duplicate_keys_and_recursive_aliases(): - malformed = parse_recipe_text("name: [unterminated") - unsafe = parse_recipe_text("!!python/object:builtins.object {}") - duplicate = parse_recipe_text( - source(header(), sequence()).replace("name: Pydantic spike", "name: First\nname: Second") - ) - recursive = parse_recipe_text("---\n&a {name: *a}\n") - assert {item.code for item in malformed.errors} == {"yaml-syntax-error"} - assert "unsafe-yaml" in {item.code for item in unsafe.errors} - assert "duplicate-key" in {item.code for item in duplicate.errors} - assert "recursive-alias" in {item.code for item in recursive.errors} - - -def test_file_api_empty_sources_and_require_recipe(tmp_path): - path = tmp_path / "recipe.yml" - path.write_text(source(header(), sequence()), encoding="utf-8") - assert parse_recipe_file(path).is_valid - assert parse_recipe_file(tmp_path / "missing.yml").errors[0].code == "file-read-error" - assert parse_recipe_text(None).errors[0].code == "invalid-source" - assert parse_recipe_text(" # comment only\n").errors[0].code == "empty-recipe" - result = parse_recipe_text("") - with pytest.raises(RecipeParseError) as error: - result.require_recipe() - assert error.value.diagnostics == result.diagnostics - - -def test_cross_document_semantic_rules_report_custom_codes(): - nested = common( - "SequenceStep", - sequence={"type": "internal", "name": "Missing"}, - input_mapping={ - "left": {"type": "direct", "value": [1], "indexed": True}, - "right": {"type": "direct", "value": [1, 2], "indexed": True}, - }, - output_mapping={ - "result": {"type": "passthrough"}, - "passed": {"type": "passfail"}, - }, - ) - duplicate = sequence(sequence_name="Main") - result = parse_recipe_text(source(header(), sequence([nested]), duplicate)) - assert { - "duplicate-sequence", - "unknown-sequence-reference", - "unequal-indexed-inputs", - "mixed-passthrough", - } <= {item.code for item in result.errors} - - -def test_ssh_context_and_ordering_are_semantic_rules(): - upload = STEP_EXAMPLES["SSHUploadStep"] - unclosed = sequence([upload], setup_steps=[common("SSHConnectStep")]) - result = parse_recipe_text(source(header(), unclosed)) - codes = {item.code for item in result.errors} - assert {"missing-ssh-global", "missing-ssh-credential", "missing-ssh-close"} <= codes - - before_connect = sequence([upload, common("SSHConnectStep")], teardown_steps=[common("SSHCloseStep")]) - assert "missing-ssh-connect" in { - item.code for item in parse_recipe_text(source(header(), before_connect)).errors - } - - -@pytest.mark.parametrize( - "path", - sorted( - path for path in RECIPES.glob("*.yml") - if path.name != "subsequence_executions_draft.yml" - ), -) -def test_normalized_bundled_corpus_passes_as_v2(path): - legacy = parse_v1(path) - assert legacy.is_valid, legacy.errors - normalized = dump_v1(legacy.require_recipe()).replace( - "recipe_version: 1.0.0", "recipe_version: 2.0.0" - ) - result = parse_recipe_text(normalized, f"normalized:{path.name}") - assert result.is_valid, result.errors - assert parse_recipe_text(dump_recipe(result.require_recipe())).recipe == result.recipe - - -def test_raw_legacy_corpus_exposes_migration_diagnostics(): - results = [ - parse_recipe_file(path) - for path in RECIPES.glob("*.yml") - if path.name != "subsequence_executions_draft.yml" - ] - assert all("unsupported-recipe-version" in {item.code for item in result.errors} for result in results) - all_codes = {item.code for result in results for item in result.errors} - assert {"noncanonical-step-type", "missing-input-type", "removed-sequence-field"} <= all_codes - - -def test_generated_schema_and_reference_are_complete_and_current(): - schema_text = render_json_schema() - schema = json.loads(schema_text) - reference = render_reference(schema) - assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") - assert reference == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") - definitions = schema["$defs"] - for model in STEP_MODELS + INPUT_MODELS + OUTPUT_MODELS: - assert model.__name__ in definitions - kind = model.model_fields.get("steptype") or model.model_fields["type"] - anchor_kind = kind.examples[0].lower() - group = "step" if model in STEP_MODELS else "input" if model in INPUT_MODELS else "output" - assert reference.count(f".. _recipe-v2-{group}-{anchor_kind}:") == 1 - - report = STEP_MODELS[0].model_fields["skip"] - assert report.description in reference - assert 'default ``false``' in reference - assert 'Example: ``false``.' in reference - - -def test_spike_has_no_runtime_gui_yamview_or_sphinx_imports(): - imported = set() - for path in Path(__file__).parent.glob("*.py"): - if path.name.startswith("test_"): - continue - tree = ast.parse(path.read_text(encoding="utf-8")) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imported.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - imported.add(node.module) - forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} - assert not any( - name == item or name.startswith(item + ".") - for name in imported - for item in forbidden - ) diff --git a/src/pypts/YamVIEW/customGUIModules.py b/src/pypts/YamVIEW/customGUIModules.py index f01a52e..c1c0ae0 100644 --- a/src/pypts/YamVIEW/customGUIModules.py +++ b/src/pypts/YamVIEW/customGUIModules.py @@ -285,7 +285,7 @@ def get_data(self): return { 'name': self.name_field.text(), 'version': "0.0.1", # hardcoded - 'recipe_version': "1.0.0", # hardcoded + 'recipe_version': "2.0.0", 'description': self.description_field.text(), 'main_sequence': self.sequence_name_field.text(), 'num_steps': self.num_steps_field.value(), @@ -344,8 +344,8 @@ def generate_template_yaml(self, data): 'steptype': 'UserInteractionStep', 'step_name': f'Setupstep', 'description': f'Empty setupstep', - 'skip': 'False', - 'continue_on_error': 'true', + 'skip': False, + 'continue_on_error': True, 'input_mapping': { 'message': {'type': 'direct', 'value': 'Tell the cook what to do'}, 'options': {'type': 'direct', 'value': [{'yes': ''}, {'no': ''}]} @@ -360,8 +360,8 @@ def generate_template_yaml(self, data): 'steptype': 'UserInteractionStep', 'step_name': f'Step {i + 1}', 'description': f'Step {i + 1} description', - 'skip': 'False', - 'continue_on_error': 'true', + 'skip': False, + 'continue_on_error': True, 'input_mapping': { 'message': {'type': 'direct', 'value': 'Tell the cook what to do'}, 'options': {'type': 'direct', 'value': [{'yes': ''}, {'no': ''}]} @@ -375,8 +375,8 @@ def generate_template_yaml(self, data): 'steptype': 'UserInteractionStep', 'step_name': f'Teardownstep', 'description': f'Empty teardown step. Doesnt do anything', - 'skip': 'False', - 'continue_on_error': 'true', + 'skip': False, + 'continue_on_error': True, 'input_mapping': { 'message': {'type': 'direct', 'value': 'This is where you functions that are required to run after every test, even if it fails in the middle. fx closing SSH connection'}, 'options': {'type': 'direct', 'value': [{'yes': ''}, {'no': ''}]} @@ -396,7 +396,13 @@ def generate_template_yaml(self, data): # REUSE-IgnoreEnd # Generate YAML - yaml_body = yaml.dump_all([header, sequence], sort_keys=False) + from pypts.recipe_language import Recipe as RecipeDefinition + from pypts.recipe_parser import dump_recipe + + definition = RecipeDefinition.model_validate( + {"header": header, "sequences": [sequence]} + ) + yaml_body = dump_recipe(definition) # Combine SPDX header and YAML body yaml_string = spdx_header + yaml_body diff --git a/src/pypts/YamVIEW/recipe_creator.py b/src/pypts/YamVIEW/recipe_creator.py index 837284f..4ba8ab1 100644 --- a/src/pypts/YamVIEW/recipe_creator.py +++ b/src/pypts/YamVIEW/recipe_creator.py @@ -38,8 +38,9 @@ import webbrowser from pypts.YamVIEW.styles import * from pypts.YamVIEW.verify_recipe import * +from pypts.recipe_parser import dump_recipe, parse_recipe_text import sys -from PySide6.QtGui import QTextCharFormat, QFont +from PySide6.QtGui import QColor, QTextCharFormat, QFont from PySide6.QtCore import Qt from PySide6.QtGui import QKeySequence, QShortcut from pypts.YamVIEW.recipe_sequencer_setup import * @@ -298,6 +299,23 @@ def highlight_line(self, line_num): self.yaml_viewer.ensureLineVisible(line_num) pass + def highlight_diagnostic(self, diagnostic): + """Highlight the exact parser span for one editor diagnostic.""" + if diagnostic.span is None: + return + selection = QTextEdit.ExtraSelection() + cursor = self.yaml_viewer.textCursor() + cursor.setPosition(diagnostic.span.start.offset) + cursor.setPosition( + max(diagnostic.span.start.offset + 1, diagnostic.span.end.offset), + QTextCursor.MoveMode.KeepAnchor, + ) + selection.cursor = cursor + selection.format.setBackground(QColor("#ffb3b3")) + self.yaml_viewer.setExtraSelections([selection]) + self.yaml_viewer.setTextCursor(cursor) + self.yaml_viewer.ensureCursorVisible() + def update_yaml_viewer(self): self.temporary_recipe_contents = self.sanitize_booleans(self.temporary_recipe_contents) self.yaml_viewer.setText(self.temporary_recipe_contents) @@ -510,9 +528,9 @@ def on_save_as_clicked(self): # Validate recipe validation_result, description = self.validate_temporary_recipe_contents() if not validation_result: - if not self.ask_save_invalid_file(): - self.log("⚠️ Save aborted.") - return + self.log("⚠️ Canonical save is blocked until the recipe validates as version 2.0.0.") + self.log(description) + return except Exception as e: self.log(f"❌ Recipe validation failed: {e}") @@ -533,7 +551,9 @@ def on_save_as_clicked(self): # Extract data from the text view try: # data = self.extract_treeView_to_data() - data = self.temporary_recipe_contents + data = dump_recipe( + parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() + ) except Exception as e: raise RuntimeError(f"Failed to extract data from the text editor: {e}") try: @@ -569,9 +589,9 @@ def on_save_clicked(self): # Validate recipe validation_result, description = self.validate_temporary_recipe_contents() if not validation_result: - if not self.ask_save_invalid_file(): - self.log("⚠️ Save aborted.") - return + self.log("⚠️ Canonical save is blocked until the recipe validates as version 2.0.0.") + self.log(description) + return if not self.current_file_path: self.log("⚠️ Save aborted, no YAML file selected.") @@ -580,7 +600,9 @@ def on_save_clicked(self): # Extract data from text view try: # data = self.extract_treeView_to_data() - data = self.temporary_recipe_contents + data = dump_recipe( + parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() + ) except Exception as e: raise RuntimeError(f"Failed to extract data from the text view: {e}") @@ -787,9 +809,12 @@ def validate_yaml_documents(self): return validation_result def validate_temporary_recipe_contents(self): + parsed = parse_recipe_text(self.temporary_recipe_contents, "") result, description = validate_recipe_string_variable(self.temporary_recipe_contents) if result == True: self.last_valid_recipe = self.temporary_recipe_contents + elif parsed.diagnostics and parsed.diagnostics[0].span is not None: + self.highlight_diagnostic(parsed.diagnostics[0]) return result, description def open_recipe(self): @@ -826,24 +851,30 @@ def load_yaml_recipe(self, file_path): self.yaml_parser.preserve_quotes = True + parsed = parse_recipe_text(raw_text, file_path) self.enable_recipe_verification = False self.update_yaml_viewer() - self.update_yaml_treeview() self.enable_recipe_verification = True - self.log(f"✅ Loaded {len(self.yaml_documents)} document(s) from: {file_path}") - try: - self.temporary_recipe_contents = self.yaml_viewer.toPlainText() - validation_result, description = self.validate_temporary_recipe_contents() - if (validation_result == True): + validation_result = parsed.is_valid + description = "\n".join(format_diagnostic(item) for item in parsed.diagnostics) + if validation_result: + self.update_yaml_treeview() + self.last_valid_recipe = raw_text self.show_recipe_ok("✅ Recipe is valid") else: + self.yaml_documents = [] + self.sequencer.set_yaml_data([]) self.show_recipe_error("Opened recipe is invalid!") self.log(description) + if parsed.diagnostics and parsed.diagnostics[0].span is not None: + self.highlight_diagnostic(parsed.diagnostics[0]) except Exception as e: self.log(f"❌ YAML verification failed, {e}") + self.log(f"Loaded recipe text from: {file_path}") + self.stacked_layout.setCurrentIndex(1) self.close_recipe.setEnabled(True) # 🔒 Enable it @@ -1025,7 +1056,7 @@ def strip_star_prefix(self, data): # done 1.0 - fix the small gui imperfections # done 1.0 - Some gui and UX improvements # done 1.0 - Create new recipe from the template -# done 1.0 - create a new helper file recipe_rules.py, where we can have yaml field type descriptions etc +# done 1.0 - derive YAML field descriptions from the production schema # done 1.0 - Recipe interactive generator - one sequence, multiple steps # done 1.0 - increase 1st column size # done 1.0 - easy way to set the YamView application diff --git a/src/pypts/YamVIEW/recipe_rules.py b/src/pypts/YamVIEW/recipe_rules.py deleted file mode 100644 index 3b190d9..0000000 --- a/src/pypts/YamVIEW/recipe_rules.py +++ /dev/null @@ -1,35 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later - -# Define required fields and expected types for top-level recipe sections -RECIPE_HEADER_REQUIRED_FIELDS = { - "version": str, - "description": str, - "globals": dict, -} - -RECIPE_SEQUENCE_REQUIRED_FIELDS = { - "description": str, - "setup_steps": list, - "steps": list, # we expect steps subsection, validated separately - "teardown_steps": list, - "parameters": dict, - "outputs": dict, - "locals": dict, -} - -# Define required fields for step types -STEP_REQUIRED_FIELDS = { - "userinteractionstep": ["steptype", "step_name", "description"], - "waitstep": ["steptype", "step_name", "description"], - "pythonmodulestep": ["steptype", "step_name", "action_type", "module", "method_name", "description"], - "userloadingstep" : ["steptype", "step_name", "description"], - "userrunmethodstep": ["steptype", "step_name", "action_type", "module", "description"], - "userwritestep": ["steptype", "step_name", "description"], - "sshconnectstep": ["steptype", "step_name", "description"], - "sshclosestep": ["steptype", "step_name", "description"], - "sshuploadstep": ["steptype", "step_name", "files", "description"], - "serialnumberstep": ["steptype", "step_name", "description"], - "default": ["steptype", "step_name", "action_type", "module", "method_name", "description"], -} diff --git a/src/pypts/YamVIEW/recipe_sequencer_setup.py b/src/pypts/YamVIEW/recipe_sequencer_setup.py index a1510d6..085e682 100644 --- a/src/pypts/YamVIEW/recipe_sequencer_setup.py +++ b/src/pypts/YamVIEW/recipe_sequencer_setup.py @@ -345,68 +345,22 @@ def edit_step(self, step_data): self.current_setup_window.close() self.current_setup_window.deleteLater() self.current_setup_window = None - # Identify type and extract node data + # Existing canonical data is rendered by the same schema-driven form + # used for new steps. node = step_data.get("_node", {}) - step_type = node.get("steptype").lower() - step_name = node.get("step_name", "Unnamed Step") step_id = step_data.get("_id", None) self.current_setup_window = Step_setup() self.current_setup_window.AlreadyID = step_id - match step_type: - case "pythonmodulestep": - self.loaded_step_parameters(step_name, node=node, method =node.get("action_type", "method"), gui_name="PythonModuleStep") - self.current_setup_window.input_mapping_widget.no_extraSteps = True - self.current_setup_window.input_mapping_widget.load_existing_data(node.get("input_mapping", {})) - self.current_setup_window.output_mapping_widget.__dict__.update(Output = True) - self.current_setup_window.output_mapping_widget.load_existing_data(node.get("output_mapping", {})) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "userinteractionstep": - self.loaded_step_parameters(step_name, node=node, gui_name="UserInteractionStep") - self.current_setup_window.input_mapping_widget.__dict__.update(allow_message=True, allow_image=True, allow_options=True, no_extraSteps=True) - self.current_setup_window.input_mapping_widget.load_existing_data(node.get("input_mapping", {})) - self.current_setup_window.output_mapping_widget.__dict__.update(Output = True, no_extraSteps=False) - self.current_setup_window.output_mapping_widget.load_existing_data(node.get("output_mapping", {})) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "waitstep": - self.loaded_step_parameters(step_name, node=node, gui_name="WaitStep") - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "userloadingstep": - self.loaded_step_parameters(step_name, node=node, gui_name="UserLoadingStep") - self.current_setup_window.input_mapping_widget.__dict__.update(allow_message=True, allow_image=True, allow_options=True, no_extraSteps=True) - self.current_setup_window.input_mapping_widget.load_existing_data(node.get("input_mapping", {})) - self.current_setup_window.output_mapping_widget.__dict__.update(loader=True, no_extraSteps=False, specific_method="passfail") - self.current_setup_window.output_mapping_widget.load_existing_data(node.get("output_mapping", {})) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "userrunmethodstep": - self.loaded_step_parameters(step_name, node=node, gui_name="UserRunMethodStep") - self.current_setup_window.input_mapping_widget.__dict__.update(allow_message=True, allow_image=True, allow_options=True, allow_method = True) - self.current_setup_window.input_mapping_widget.load_existing_data(node.get("input_mapping", {})) - self.current_setup_window.output_mapping_widget.__dict__.update(Output = True) - self.current_setup_window.output_mapping_widget.load_existing_data(node.get("output_mapping", {})) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "userwritestep": - self.loaded_step_parameters(step_name, node=node, gui_name="UserWriteStep") - #self.current_setup_window.input_mapping_widget.__dict__.update(allow_message=True, allow_image=True, allow_options=True, allow_method = True) - #self.current_setup_window.input_mapping_widget.load_existing_data(node.get("input_mapping", {})) - #self.output_mapping_widget = self.InOutputMappingWidget( output = True) - #self.current_setup_window.output_mapping_widget.load_existing_data(node.get("output_mapping", {})) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - case "sshconnectstep": - self.current_setup_window.clear_layout(self.current_setup_window.step_specific_container) - self.loaded_step_parameters(step_name, node=node,gui_name="SSHConnectStep") - self.current_setup_window.load_existing_globals(data=self.preamble_globals) - self.current_setup_window.finished.connect(lambda result: self.on_edit_window_closed(result)) - self.current_setup_window.show() - - case _: - QMessageBox.warning(self, "Unsupported Step", f"Editing not yet supported for '{step_type}'.") + try: + self.current_setup_window.load_definition(node) + except (KeyError, ValueError) as error: + QMessageBox.warning(self, "Invalid Step", str(error)) + return + self.current_setup_window.finished.connect( + lambda result: self.on_edit_window_closed(result) + ) + self.current_setup_window.show() def on_edit_window_closed(self, result): if result == QDialog.Accepted: diff --git a/src/pypts/YamVIEW/recipe_step_setup.py b/src/pypts/YamVIEW/recipe_step_setup.py index 2840e38..a94030f 100644 --- a/src/pypts/YamVIEW/recipe_step_setup.py +++ b/src/pypts/YamVIEW/recipe_step_setup.py @@ -9,6 +9,276 @@ QPushButton,QLineEdit,QTextEdit,QCheckBox, QHBoxLayout, QMessageBox,QWidget,QTableWidgetItem,QTableWidget, QAbstractItemView, QScrollArea) import os, uuid +import json +from typing import Any + +from pydantic import TypeAdapter + +from pypts.recipe_language import Recipe as RecipeDefinition +from pypts.recipe_language import Step as AuthorableStepDefinition + + +def recipe_form_schema() -> dict[str, Any]: + """Return the production aggregate schema used to build YamVIEW forms.""" + return RecipeDefinition.model_json_schema(by_alias=True, mode="validation") + + +def resolve_schema(schema: dict[str, Any], node: dict[str, Any]) -> dict[str, Any]: + """Resolve a local JSON Schema reference without a parallel registry.""" + while "$ref" in node: + prefix = "#/$defs/" + reference = node["$ref"] + if not reference.startswith(prefix): + raise ValueError(f"Unsupported schema reference: {reference}") + node = schema["$defs"][reference.removeprefix(prefix)] + return node + + +def discriminator_schemas(name: str, schema: dict[str, Any] | None = None): + """Map discriminator values to their resolved model schemas.""" + schema = schema or recipe_form_schema() + definition = schema["$defs"][name] + mapping = definition["discriminator"]["mapping"] + return {key: resolve_schema(schema, {"$ref": reference}) for key, reference in mapping.items()} + + +def schema_widget_kind(field: dict[str, Any]) -> str: + """Select a primitive or structured editor solely from JSON Schema type.""" + if "enum" in field or "const" in field: + return "choice" + variants = field.get("anyOf", []) + kinds = {item.get("type") for item in variants} + kind = field.get("type") + if kind == "boolean": + return "boolean" + if kind in {"integer", "number"}: + return "number" + if kind == "string": + return "text" + if kind in {"array", "object"} or kinds & {"array", "object"} or not (kind or kinds): + return "structured" + return "text" + + +def recipe_form_description(schema: dict[str, Any] | None = None) -> dict[str, Any]: + """Describe selectors and fields directly from the production JSON Schema.""" + schema = schema or recipe_form_schema() + + def variants(name: str): + result = {} + for discriminator, definition in discriminator_schemas(name, schema).items(): + required = set(definition.get("required", [])) + result[discriminator] = { + "title": definition.get("title", discriminator), + "description": definition.get("description", ""), + "fields": { + field_name: { + **field_schema, + "required": field_name in required, + "widget": schema_widget_kind(field_schema), + } + for field_name, field_schema in definition.get("properties", {}).items() + }, + } + return result + + return { + "steps": variants("Step"), + "inputs": variants("InputMapping"), + "outputs": variants("OutputMapping"), + } + + +def build_schema_widget(field: dict[str, Any], parent=None): + """Build a primitive or structured editor from one resolved field schema.""" + mapping_reference = field.get("additionalProperties", {}).get("$ref", "") + if mapping_reference.endswith("/InputMapping"): + return DiscriminatedMappingWidget("inputs", parent) + if mapping_reference.endswith("/OutputMapping"): + return DiscriminatedMappingWidget("outputs", parent) + kind = schema_widget_kind(field) + if kind == "choice": + widget = QComboBox(parent) + values = field.get("enum", [field.get("const")]) + for value in values: + widget.addItem(str(value), value) + elif kind == "boolean": + widget = QCheckBox(parent) + widget.setChecked(bool(field.get("default", False))) + elif kind == "structured": + widget = QTextEdit(parent) + if "default" in field: + widget.setPlainText(json.dumps(field["default"], ensure_ascii=False)) + widget.setMaximumHeight(90) + else: + widget = QLineEdit(parent) + if "default" in field and field["default"] is not None: + widget.setText(str(field["default"])) + details = field.get("description", "") + if field.get("examples"): + details += f" Example: {field['examples'][0]!r}." + widget.setToolTip(details) + return widget + + +class SchemaFormWidget(QWidget): + """Generic YamVIEW form whose fields come entirely from JSON Schema.""" + + def __init__(self, variant: dict[str, Any], parent=None): + super().__init__(parent) + self.variant = variant + self.field_widgets = {} + layout = QVBoxLayout(self) + description = variant.get("description") + if description: + layout.addWidget(QLabel(description)) + for name, field in variant.get("fields", {}).items(): + suffix = " *" if field.get("required") else "" + label = QLabel(f"{name}{suffix}") + label.setToolTip(field.get("description", "")) + widget = build_schema_widget(field, self) + self.field_widgets[name] = widget + layout.addWidget(label) + layout.addWidget(widget) + + def values(self) -> dict[str, Any]: + """Read authorable values from the schema-selected controls.""" + values = {} + fields = self.variant["fields"] + for name, widget in self.field_widgets.items(): + field = fields[name] + if isinstance(widget, DiscriminatedMappingWidget): + value = widget.values() + elif isinstance(widget, QComboBox): + value = widget.currentData() + elif isinstance(widget, QCheckBox): + value = widget.isChecked() + elif isinstance(widget, QTextEdit): + text = widget.toPlainText().strip() + if not text: + if "default" not in field and not field.get("required"): + continue + value = field.get("default") + else: + value = json.loads(text) + else: + text = widget.text().strip() + if not text and not field.get("required"): + continue + value = text + if field.get("type") in {"integer", "number"}: + value = json.loads(text) + values[name] = value + return values + + def load_values(self, values: dict[str, Any]) -> None: + """Populate controls from an existing canonical step mapping.""" + for name, value in values.items(): + widget = self.field_widgets.get(name) + if widget is None: + continue + if isinstance(widget, DiscriminatedMappingWidget): + widget.load_values(value) + elif isinstance(widget, QComboBox): + index = widget.findData(value) + if index >= 0: + widget.setCurrentIndex(index) + elif isinstance(widget, QCheckBox): + widget.setChecked(bool(value)) + elif isinstance(widget, QTextEdit): + widget.setPlainText(json.dumps(value, ensure_ascii=False, indent=2)) + else: + widget.setText("" if value is None else str(value)) + + +class DiscriminatedMappingWidget(QWidget): + """Editable mapping rows driven by an input/output discriminator schema.""" + + def __init__(self, mapping_kind: str, parent=None): + super().__init__(parent) + self.variants = recipe_form_description()[mapping_kind] + self.rows = [] + self.layout = QVBoxLayout(self) + add_button = QPushButton("Add mapping") + add_button.clicked.connect(lambda: self.add_row()) + self.layout.addWidget(add_button) + + @staticmethod + def _clear(layout): + while layout.count(): + item = layout.takeAt(0) + if item.widget() is not None: + item.widget().deleteLater() + elif item.layout() is not None: + DiscriminatedMappingWidget._clear(item.layout()) + + def add_row(self, name="", value=None): + value = value or {} + row_widget = QWidget(self) + row_layout = QVBoxLayout(row_widget) + heading = QHBoxLayout() + name_edit = QLineEdit(row_widget) + name_edit.setPlaceholderText("mapping name") + name_edit.setText(name) + type_combo = QComboBox(row_widget) + type_combo.addItems(self.variants) + mapping_type = value.get("type") + if mapping_type in self.variants: + type_combo.setCurrentText(mapping_type) + remove_button = QPushButton("Remove", row_widget) + heading.addWidget(name_edit) + heading.addWidget(type_combo) + heading.addWidget(remove_button) + row_layout.addLayout(heading) + form_layout = QVBoxLayout() + row_layout.addLayout(form_layout) + row = { + "widget": row_widget, + "name": name_edit, + "type": type_combo, + "form_layout": form_layout, + "form": None, + "value": value, + } + self.rows.append(row) + self.layout.insertWidget(self.layout.count() - 1, row_widget) + + def render(discriminator): + self._clear(form_layout) + form = SchemaFormWidget(self.variants[discriminator], row_widget) + row["form"] = form + form_layout.addWidget(form) + if row["value"].get("type") == discriminator: + form.load_values(row["value"]) + row["value"] = {} + + def remove(): + self.rows.remove(row) + row_widget.setParent(None) + row_widget.deleteLater() + + type_combo.currentTextChanged.connect(render) + remove_button.clicked.connect(remove) + render(type_combo.currentText()) + + def values(self): + result = {} + for row in self.rows: + name = row["name"].text().strip() + if not name: + raise ValueError("Mapping rows require a name.") + value = row["form"].values() + value["type"] = row["type"].currentText() + result[name] = value + return result + + def load_values(self, values): + for row in tuple(self.rows): + self.rows.remove(row) + row["widget"].setParent(None) + row["widget"].deleteLater() + for name, value in values.items(): + self.add_row(name, value) class Sequence_setup(QDialog): def __init__(self, steps, parent=None): @@ -243,33 +513,25 @@ def __init__(self,use_input_mapping=True, use_output_mapping=True, parent=None): layout.addWidget(QLabel("Steptype")) self.list_steptype = QComboBox() - self.steptypes = [ - "PythonModuleStep", - "UserInteractionStep", - "WaitStep", - "UserLoadingStep", - "UserRunMethodStep", - "UserWriteStep", - "SSHConnectStep", - "SSHCloseStep" - ] + self.steptypes = list(discriminator_schemas("Step")) self.list_steptype.addItems(self.steptypes) layout.addWidget(self.list_steptype) self.list_steptype.currentTextChanged.connect(self.on_step_type_changed) self._previous_step_type = self.list_steptype.currentText() self._skip_warning = False + self.form_description = recipe_form_description() + self.schema_form = None # Container for step-specific widgets self.step_specific_container = QVBoxLayout() container_widget = QWidget() container_widget.setLayout(self.step_specific_container) - self.setup_pythonmodulestep() + self._render_schema_step(self.list_steptype.currentText()) scroll = QScrollArea() scroll.setWidgetResizable(True) scroll.setWidget(container_widget) - self.step_specific_container.addStretch() layout.addWidget(scroll) # OK/Cancel buttons @@ -315,10 +577,22 @@ def on_step_type_changed(self, step_type: str): self._previous_step_type = step_type # Clear previous widgets self.clear_layout(self.step_specific_container) - # Call the corresponding setup function - func_name = f"setup_{step_type.lower()}" - if hasattr(self, func_name): - getattr(self, func_name)() + self._render_schema_step(step_type) + + def _render_schema_step(self, step_type: str): + """Render the selected authorable step directly from JSON Schema.""" + self.schema_form = SchemaFormWidget(self.form_description["steps"][step_type]) + self.step_specific_container.addWidget(self.schema_form) + self.step_specific_container.addStretch() + + def load_definition(self, node: dict[str, Any]) -> None: + """Load an existing canonical step into the schema-driven editor.""" + step_type = node["steptype"] + self._skip_warning = True + self.list_steptype.setCurrentText(step_type) + self._skip_warning = False + self.schema_form.load_values(node) + self.setWindowTitle(f"Edit Step: {node.get('step_name', step_type)}") def clear_layout(self, layout): while layout.count(): @@ -676,6 +950,34 @@ def accept(self): else: StepID = str(uuid.uuid4()) + try: + authorable = self.schema_form.values() + authorable["steptype"] = step_type + definition = TypeAdapter(AuthorableStepDefinition).validate_python(authorable) + except Exception as error: + self.setStyleSheet("""QMessageBox QPushButton { color: black;}""") + QMessageBox.warning(self, "Invalid step", str(error)) + return + + node = definition.model_dump(mode="python", by_alias=True, exclude_none=True) + self.result_step = { + "steptype": step_type, + "step_name": definition.step_name, + "_parent": "main_folder", + "_node": node, + "_id": StepID, + } + self.global_variables = {} + self.local_variables = {} + globals_in, locals_in = self.extract_locals_globals(node["input_mapping"]) + globals_out, locals_out = self.extract_locals_globals(node["output_mapping"]) + self.global_variables.update(globals_in) + self.global_variables.update(globals_out) + self.local_variables.update(locals_in) + self.local_variables.update(locals_out) + super().accept() + return + validate_method = f"validate_{step_type.lower()}" if hasattr(self, validate_method): ok, error_msg = getattr(self, validate_method)() @@ -1128,20 +1430,8 @@ def add_row(self): # direct/global/local row["type_combo"] = QComboBox() - if self.Output: - types = ["equals", "passthrough", "passfail", "range", "global", "local"] - elif self.loader: - types = ["global", "local"] - else: - types = ["direct", "global", "local"] - if self.allow_message: - types.append("message") - if self.allow_image: - types.append("image_path") - if self.allow_options: - types.append("options") - if self.allow_method: - types.append("method") + mapping_name = "OutputMapping" if self.Output else "InputMapping" + types = list(discriminator_schemas(mapping_name)) row["type_combo"].addItems(types) diff --git a/src/pypts/YamVIEW/verify_recipe.py b/src/pypts/YamVIEW/verify_recipe.py index ed11046..6780042 100644 --- a/src/pypts/YamVIEW/verify_recipe.py +++ b/src/pypts/YamVIEW/verify_recipe.py @@ -1,445 +1,105 @@ # SPDX-FileCopyrightText: 2025 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -from pathlib import Path -import os -import yaml -from pypts.YamVIEW.recipe_rules import RECIPE_HEADER_REQUIRED_FIELDS, RECIPE_SEQUENCE_REQUIRED_FIELDS, STEP_REQUIRED_FIELDS +"""YamVIEW compatibility wrappers around the production recipe parser.""" -SUPPORTED_STEP_TYPES = { - "indexedstep", "pythonmodulestep", "sequencestep", "userinteractionstep", - "waitstep", "userloadingstep", "userrunmethodstep", "userwritestep", - "serialnumberstep", "sshconnectstep", "sshclosestep", "sshuploadstep", -} -VERDICT_TYPES = {"passthrough", "passfail", "equals", "range"} +from __future__ import annotations -class RecipeValidationError(Exception): - def __init__(self, faults, warnings): - self.faults = faults - self.warnings = warnings - super().__init__(f"Validation failed with {len(faults)} faults and {len(warnings)} warnings") +from collections.abc import Iterable +from pathlib import Path -def extract_line_map(node, path=()): - result = {} - if isinstance(node, yaml.MappingNode): - for key_node, value_node in node.value: - key = key_node.value - new_path = path + (key,) - result[new_path] = key_node.start_mark.line + 1 - result.update(extract_line_map(value_node, new_path)) - elif isinstance(node, yaml.SequenceNode): - for idx, item_node in enumerate(node.value): - new_path = path + (idx,) - result.update(extract_line_map(item_node, new_path)) - return result +from pypts.recipe_parser import Diagnostic, parse_recipe_file, parse_recipe_text -def validate_field(doc, field_name, expected_type, faults, warnings, context, line_map, path=()): - full_path = path + (field_name,) - line_info = f"(line {line_map.get(full_path, '?')})" - if field_name not in doc: - faults.append(f"[{context}] Missing required field: '{field_name}' {line_info}") - return +def format_diagnostic(diagnostic: Diagnostic) -> str: + """Format one structured parser finding for display in YamVIEW.""" + source = diagnostic.source_name or "" + location = source + if diagnostic.span is not None: + start = diagnostic.span.start + end = diagnostic.span.end + location += f":{start.line}:{start.column}-{end.line}:{end.column}" + path = "".join( + f"[{part}]" if isinstance(part, int) else (f".{part}" if index else str(part)) + for index, part in enumerate(diagnostic.path) + ) + if path: + location += f" ({path})" + return f"[{diagnostic.code}] {location}: {diagnostic.message}" - value = doc[field_name] - if value is None: - if expected_type == str: - warnings.append(f"[{context}] Field '{field_name}' is null {line_info}") - else: - faults.append(f"[{context}] Field '{field_name}' is null but expected type {expected_type.__name__} {line_info}") - return +class RecipeValidationError(Exception): + """Compatibility exception containing formatted production diagnostics.""" + + def __init__( + self, + faults: Iterable[str], + warnings: Iterable[str] = (), + diagnostics: Iterable[Diagnostic] = (), + ): + self.faults = list(faults) + self.warnings = list(warnings) + self.diagnostics = tuple(diagnostics) + message = f"Validation failed with {len(self.faults)} faults and {len(self.warnings)} warnings" + if self.faults or self.warnings: + message += ":\n" + "\n".join((*self.faults, *self.warnings)) + super().__init__(message) + + +def _raise_for_diagnostics(diagnostics: tuple[Diagnostic, ...]) -> None: + faults = [format_diagnostic(item) for item in diagnostics if item.severity == "error"] + warnings = [format_diagnostic(item) for item in diagnostics if item.severity == "warning"] + if faults or warnings: + raise RecipeValidationError(faults, warnings, diagnostics) - if not isinstance(value, expected_type): - faults.append( - f"[{context}] Field '{field_name}' should be of type {expected_type.__name__}, " - f"but got {type(value).__name__} {line_info}" - ) - return - if expected_type == str and not value.strip(): - warnings.append(f"[{context}] Field '{field_name}' is an empty string {line_info}") +def validate_recipe_file(filepath) -> None: + """Raise :class:`RecipeValidationError` unless *filepath* is valid v2.""" + result = parse_recipe_file(filepath) + _raise_for_diagnostics(result.diagnostics) -def validate_step_fields(steps, faults, line_map, base_path=()): - if not isinstance(steps, list): - faults.append(f"[{'.'.join(map(str, base_path))}] should be a list") - return - for idx, step in enumerate(steps): - if not isinstance(step, dict): - faults.append(f"[Step {idx}] Step is not a dictionary") - continue - step_path = base_path + (idx,) - step_line = line_map.get(step_path, '?') - step_name = step.get("step_name", f"at index {idx}") - context = f"Step {idx} ({step_name})" +def validate_recipe_filepath(file_path) -> bool: + """Return whether *file_path* contains a valid recipe-language 2 recipe.""" + try: + validate_recipe_file(file_path) + except RecipeValidationError: + return False + return True - steptype = step.get("steptype") - steptype_key = steptype.lower() if isinstance(steptype, str) else "" - if steptype_key not in SUPPORTED_STEP_TYPES: - faults.append(f"[{context}] Unknown step type '{steptype}'") - required_fields = STEP_REQUIRED_FIELDS.get(steptype_key, STEP_REQUIRED_FIELDS["default"]) - for field in required_fields: - field_path = step_path + (field,) - line = line_map.get(field_path, step_line) - if field not in step: - faults.append(f"[{context}] Missing required field: '{field}' (line {line})") +def validate_recipe_string_variable(content: str) -> tuple[bool, str]: + """Validate edited YAML text and return YamVIEW's historical tuple result.""" + result = parse_recipe_text(content, "") + if result.diagnostics: + messages = "\n".join(format_diagnostic(item) for item in result.diagnostics) + return False, messages + return True, "Validation passed for the variable recipe." - # Validate that input_mapping and output_mapping are dictionaries - for field in ("input_mapping", "output_mapping"): - if field in step: - value = step[field] - line = line_map.get(step_path + (field,), step_line) - if value is None: - faults.append(f"[{context}] Field '{field}' is null, expected dict (line {line})") - elif not isinstance(value, dict): - faults.append(f"[{context}] Field '{field}' should be a dictionary but got {type(value).__name__} (line {line})") - input_mapping = step.get("input_mapping", {}) - if isinstance(input_mapping, dict): - indexed_lengths = [] - for input_name, config in input_mapping.items(): - if not isinstance(config, dict): - faults.append(f"[{context}] Input '{input_name}' mapping should be a dictionary") - continue - if config.get("indexed"): - if not isinstance(config.get("value"), list): - faults.append(f"[{context}] Indexed input '{input_name}' value should be a list") - else: - indexed_lengths.append(len(config["value"])) - if indexed_lengths and len(set(indexed_lengths)) > 1: - faults.append(f"[{context}] Indexed input lists must have equal lengths") +validate_recipe_string = validate_recipe_string_variable - output_mapping = step.get("output_mapping", {}) - if isinstance(output_mapping, dict): - verdicts = [] - for output_name, config in output_mapping.items(): - if not isinstance(config, dict): - faults.append(f"[{context}] Output '{output_name}' mapping should be a dictionary") - continue - mapping_type = config.get("type") - if not isinstance(mapping_type, str): - faults.append(f"[{context}] Output '{output_name}' is missing string field 'type'") - continue - if mapping_type in VERDICT_TYPES: - verdicts.append(mapping_type) - required = {"equals": ("value",), "range": ("min", "max"), - "local": ("local_name",), "global": ("global_name",)} - for required_field in required.get(mapping_type, ()): - if required_field not in config: - faults.append( - f"[{context}] Output '{output_name}' type '{mapping_type}' " - f"requires '{required_field}'" - ) - if "passthrough" in verdicts and len(verdicts) != 1: - faults.append(f"[{context}] passthrough must be the sole verdict mapping") - # Check if 'skip' is a boolean - if "skip" in step: - skip_value = step["skip"] - skip_line = line_map.get(step_path + ("skip",), step_line) - if not isinstance(skip_value, bool): - faults.append(f"[{context}] Field 'skip' should be a boolean but got {type(skip_value).__name__} (line {skip_line})") - for boolean_field in ("critical", "continue_on_error"): - if boolean_field in step and not isinstance(step[boolean_field], bool): - faults.append(f"[{context}] Field '{boolean_field}' should be a boolean") +def validate_all_recipes_in_folder(folder_path): + errors = [] + for path in Path(folder_path).iterdir(): + if path.suffix.lower() in {".yaml", ".yml"}: + try: + validate_recipe_file(path) + except RecipeValidationError as error: + errors.append((path.name, error)) + return not errors - if steptype_key == "sequencestep": - sequence = step.get("sequence") - if not isinstance(sequence, dict) or not isinstance(sequence.get("name"), str): - faults.append(f"[{context}] SequenceStep requires sequence.name") def validate_all_recipes_in_folders(folder_paths): - if isinstance(folder_paths, str): - folder_paths = [folder_paths] # allow single path input - + if isinstance(folder_paths, (str, Path)): + folder_paths = [folder_paths] errors = [] for folder_path in folder_paths: - for filename in os.listdir(folder_path): - if filename.endswith((".yaml", ".yml")): - full_path = os.path.join(folder_path, filename) + for path in Path(folder_path).iterdir(): + if path.suffix.lower() in {".yaml", ".yml"}: try: - validate_recipe_file(full_path) - except RecipeValidationError as e: - errors.append((filename, e, folder_path)) + validate_recipe_file(path) + except RecipeValidationError as error: + errors.append((path.name, error, str(folder_path))) return errors - -def validate_all_recipes_in_folder(folder_path): - errors = [] - for filename in os.listdir(folder_path): - if filename.endswith(".yaml") or filename.endswith(".yml"): - full_path = os.path.join(folder_path, filename) - try: - validate_recipe_file(full_path) - except RecipeValidationError as e: - errors.append((filename, e)) - - if errors: - print("\n❌ Summary: Some recipe files failed validation.") - for filename, e in errors: - print(f" - {filename}: {len(e.faults)} faults, {len(e.warnings)} warnings") - pass - return False - else: - print("\n✅ All recipe files validated successfully.") - return True - -def validate_recipe_filepath(file_path): - errors = [] - p = Path(file_path) - filename = p.stem - try: - validate_recipe_file(file_path) - except RecipeValidationError as e: - errors.append((filename, e)) - - if errors: - # print("\n❌ Summary: Some recipe files failed validation.") - for filename, e in errors: - print(f" - {filename}: {len(e.faults)} faults, {len(e.warnings)} warnings") - pass - return False - else: - print("\n✅ All recipe files validated successfully.") - return True - -def validate_recipe_file(filepath): - faults = [] - warnings = [] - - with open(filepath, 'r') as f: - content = f.read() - - try: - docs_nodes = list(yaml.compose_all(content)) - except yaml.YAMLError as e: - raise RecipeValidationError([f"YAML parsing error in '{filepath}': {e}"], []) - - docs = list(yaml.safe_load_all(content)) - header = next((doc for doc in docs if isinstance(doc, dict) and "name" in doc), None) - sequence_names = { - doc.get("sequence_name") for doc in docs - if isinstance(doc, dict) and isinstance(doc.get("sequence_name"), str) - } - - for i, (doc, node) in enumerate(zip(docs, docs_nodes)): - if not isinstance(doc, dict): - faults.append(f"[{filepath}, Document {i}] is not a dictionary (line {node.start_mark.line + 1})") - continue - - line_map = extract_line_map(node) - first_key = next(iter(doc), None) - - if first_key == "name": - context = f"{filepath} Header" - if "continue_on_error" in doc and not isinstance(doc["continue_on_error"], bool): - faults.append( - f"[{context}] Top-level 'continue_on_error' should be a boolean" - ) - for field, expected_type in RECIPE_HEADER_REQUIRED_FIELDS.items(): - validate_field(doc, field, expected_type, faults, warnings, context, line_map) - - elif first_key == "sequence_name": - context = f"{filepath} Sequence" - for field, expected_type in RECIPE_SEQUENCE_REQUIRED_FIELDS.items(): - # For "steps" subsection, validate presence and content separately - if field == "steps": - if field not in doc: - line_info = f"(line {line_map.get(('steps',), '?')})" - faults.append(f"[{context}] Missing required subsection: 'steps' {line_info}") - else: - validate_step_fields(doc["steps"], faults, line_map, base_path=("steps",)) - else: - validate_field(doc, field, expected_type, faults, warnings, context, line_map) - - if "locals" not in doc: - faults.append(f"[{context}] Missing 'locals' section") - elif not isinstance(doc["locals"], dict): - faults.append(f"[{context}] 'locals' should be a dictionary") - else: - line = node.start_mark.line + 1 - faults.append(f"[{filepath}, Document {i}] Unrecognized document type, first key: '{first_key}' (line {line})") - - if header is not None: - main_sequence = header.get("main_sequence", "Main") - if not isinstance(main_sequence, str): - faults.append(f"[{filepath} Header] Field 'main_sequence' should be of type str") - elif main_sequence not in sequence_names: - faults.append( - f"[{filepath} Header] Main sequence '{main_sequence}' does not exist" - ) - - for doc in docs: - if not isinstance(doc, dict) or "sequence_name" not in doc: - continue - for mapping_field in ("parameters", "outputs", "locals"): - if not isinstance(doc.get(mapping_field), dict): - faults.append( - f"[{filepath} Sequence {doc.get('sequence_name')}] " - f"'{mapping_field}' should be a dictionary" - ) - for section in ("setup_steps", "steps", "teardown_steps"): - validate_step_fields(doc.get(section, []), faults, {}, base_path=(section,)) - sections = { - name: value if isinstance(value := doc.get(name, []), list) else [] - for name in ("setup_steps", "steps", "teardown_steps") - } - all_steps = sections["setup_steps"] + sections["steps"] + sections["teardown_steps"] - for step in all_steps: - step_type = step.get("steptype") if isinstance(step, dict) else None - if isinstance(step_type, str) and step_type.casefold() == "sequencestep": - target = step.get("sequence", {}).get("name") if isinstance(step.get("sequence"), dict) else None - if target and target not in sequence_names: - faults.append(f"[Sequence {doc['sequence_name']}] references unknown sequence '{target}'") - - def step_types(section): - return [ - value.casefold() for step in section - if isinstance(step, dict) - and isinstance((value := step.get("steptype")), str) - ] - setup_types = step_types(sections["setup_steps"]) - main_types = step_types(sections["steps"]) - teardown_types = step_types(sections["teardown_steps"]) - uses_ssh = any(t.startswith("ssh") for t in setup_types + main_types + teardown_types) - if uses_ssh and header is not None: - globals_data = header.get("globals", {}) - for name in ("ssh_client", "host", "user", "port"): - if name not in globals_data: - faults.append(f"[{filepath} Header] SSH recipes require global '{name}'") - if "password" not in globals_data and "private_key" not in globals_data: - faults.append(f"[{filepath} Header] SSH recipes require 'password' or 'private_key'") - if "sshuploadstep" in main_types and "sshconnectstep" not in setup_types: - faults.append(f"[Sequence {doc['sequence_name']}] SSHUploadStep requires SSHConnectStep in setup_steps") - if "sshconnectstep" in setup_types and "sshclosestep" not in teardown_types: - faults.append(f"[Sequence {doc['sequence_name']}] SSHConnectStep requires SSHCloseStep in teardown_steps") - - if faults or warnings: - if faults: - print("🛑 Faults:") - for f in faults: - print(" -", f) - - if warnings: - print("⚠️ Warnings:") - for w in warnings: - print(" -", w) - - raise RecipeValidationError(faults, warnings) - - print(f"✅ Validation passed for '{filepath}'.") - -def validate_recipe_string_variable(content): - faults = [] - warnings = [] - - try: - docs_nodes = list(yaml.compose_all(content)) - except yaml.YAMLError as e: - - return False, f"❌ YAML parsing error: {e}" - # raise RecipeValidationError([f"❌ YAML parsing error: {e}"], []) - - docs = list(yaml.safe_load_all(content)) - header = next((doc for doc in docs if isinstance(doc, dict) and "name" in doc), None) - sequence_names = { - doc.get("sequence_name") for doc in docs - if isinstance(doc, dict) and isinstance(doc.get("sequence_name"), str) - } - - for i, (doc, node) in enumerate(zip(docs, docs_nodes)): - if not isinstance(doc, dict): - faults.append(f"[, Document {i}] is not a dictionary (line {node.start_mark.line + 1})") - continue - - line_map = extract_line_map(node) - first_key = next(iter(doc), None) - - if first_key == "name": - context = f"Header" - if "continue_on_error" in doc and not isinstance(doc["continue_on_error"], bool): - faults.append( - "[Header] Top-level 'continue_on_error' should be a boolean" - ) - for field, expected_type in RECIPE_HEADER_REQUIRED_FIELDS.items(): - validate_field(doc, field, expected_type, faults, warnings, context, line_map) - - elif first_key == "sequence_name": - context = f"Sequence" - for field, expected_type in RECIPE_SEQUENCE_REQUIRED_FIELDS.items(): - if field in ("setup_steps", "steps", "teardown_steps"): - if field not in doc: - line_info = f"(line {line_map.get((field,), '?')})" - faults.append(f"[{context}] Missing required subsection: '{field}' {line_info}") - else: - validate_step_fields(doc[field], faults, line_map, base_path=(field,)) - else: - validate_field(doc, field, expected_type, faults, warnings, context, line_map) - - if "locals" not in doc: - faults.append(f"[{context}] Missing 'locals' section") - elif not isinstance(doc["locals"], dict): - faults.append(f"[{context}] 'locals' should be a dictionary") - else: - line = node.start_mark.line + 1 - faults.append( - f"[Document {i}] Unrecognized document type, first key: '{first_key}' (line {line})") - - if header is not None: - main_sequence = header.get("main_sequence", "Main") - if not isinstance(main_sequence, str) or main_sequence not in sequence_names: - faults.append(f"[Header] Main sequence '{main_sequence}' does not exist") - - for doc in docs: - if not isinstance(doc, dict) or "sequence_name" not in doc: - continue - for section in ("setup_steps", "steps", "teardown_steps"): - for step in doc.get(section, []) if isinstance(doc.get(section, []), list) else []: - step_type = step.get("steptype") if isinstance(step, dict) else None - if isinstance(step_type, str) and step_type.casefold() == "sequencestep": - sequence = step.get("sequence") - target = sequence.get("name") if isinstance(sequence, dict) else None - if target and target not in sequence_names: - faults.append( - f"[Sequence {doc['sequence_name']}] references unknown sequence '{target}'" - ) - - output_lines = [] - - if faults or warnings: - output_lines.append("❌ Validation for recipe completed with issues:") - if faults: - output_lines.append("🛑 Faults:") - for f in faults: - output_lines.append(f" - {f}") - if warnings: - output_lines.append("⚠️ Warnings:") - for w in warnings: - output_lines.append(f" - {w}") - # pass here means continue to raise below - - return False, "\n".join(output_lines) - # raise RecipeValidationError(faults, warnings) - else: - output_lines.append("✅ Validation passed for the variable recipe.") - return True, "\n".join(output_lines) - -if __name__ == "__main__": - current_dir = os.path.dirname(__file__) # directory of current file - parent_dir = os.path.dirname(current_dir) # one directory up - - recipes_dir = os.path.join(parent_dir, "recipes") - extra_recipes_dir = os.path.join(parent_dir, "example_commented_recipes") - folders_to_validate = [recipes_dir, extra_recipes_dir] - - try: - errors = validate_all_recipes_in_folders(folders_to_validate) - if errors: - print("❌ Summary: Some recipe files failed validation!") - for filename, err, folder in errors: - print(f" - {os.path.join(folder, filename)}: {err}") - else: - print("✅ All recipe files validated successfully.") - except Exception as e: - print(f"❌ Unhandled exception while validating the recipes: {e}") diff --git a/src/pypts/recipe.py b/src/pypts/recipe.py index 6b097e5..812f5ef 100644 --- a/src/pypts/recipe.py +++ b/src/pypts/recipe.py @@ -3,9 +3,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later -from pypts.YamVIEW.verify_recipe import validate_recipe_filepath import copy -import yaml import logging from typing import List, Dict, Self from pathlib import Path @@ -20,6 +18,13 @@ import re from threading import Event from pypts.utils import WAIT_FOR_TERMINATION +from pypts.recipe_language import ( + CommonStep as AuthorableStepDefinition, + DirectInput as DirectInputDefinition, + Recipe as RecipeDefinition, + Sequence as SequenceDefinition, +) +from pypts.recipe_parser import parse_recipe_file # from pts import Runtime logger = logging.getLogger(__name__) @@ -249,113 +254,53 @@ class Recipe: execution flow. The detailed structure of the recipe YAML file is described in :doc:`yaml_format`. - Args: - recipe_file_path (str or Path): Path to the recipe YAML file. - file_loader (callable, optional): A function that takes a path and returns - an iterator over loaded YAML documents. Defaults to loading from a - local YAML file. - event_sender (callable, optional): A function to send events during recipe - execution. Takes `runtime`, `event_name`, and `*event_data` as arguments. - Defaults to using `runtime.send_event`. + Recipe files must validate as recipe language 2.0.0 before any executable + runtime state is constructed. """ - def __init__(self, recipe_file_path, file_loader=None, event_sender=None): - self.file_loader = file_loader or self._default_file_loader + def __init__(self, recipe_file_path, event_sender=None): self.event_sender = event_sender or self._default_event_sender - self.__load_recipe(recipe_file_path) - self.recipe_file_name = Path(recipe_file_path).name # Store filename - - def _default_file_loader(self, path): - """Default implementation that loads a YAML file""" - with open(path, 'r') as file: - # Read the file content into memory first - file_content = file.read() - # Then return an iterator over the YAML documents - return yaml.safe_load_all(file_content) + definition = parse_recipe_file(recipe_file_path).require_recipe() + self._load_definition(definition, str(recipe_file_path)) + + @classmethod + def from_definition( + cls, + definition: RecipeDefinition, + source_name: str = "", + event_sender=None, + ): + """Construct executable state from an already validated aggregate model.""" + if not isinstance(definition, RecipeDefinition): + raise TypeError("definition must be a validated recipe_language.Recipe") + instance = cls.__new__(cls) + instance.event_sender = event_sender or instance._default_event_sender + instance._load_definition(definition, source_name) + return instance def _default_event_sender(self, runtime, event_name, *event_data): """Default implementation that uses runtime's send_event""" runtime.send_event(event_name, *event_data) - def __load_recipe(self, recipe_file_path): - """Loads recipe data using the file_loader""" - logger.info(f"Loading recipe file {recipe_file_path}.") - self.sequences = {} - - try: - recipe_data = self.file_loader(recipe_file_path) - logger.debug(f"File loader returned recipe_data type: {type(recipe_data)}") - - recipe_main_data = next(recipe_data) - logger.debug(f"Recipe main data keys: {recipe_main_data.keys() if isinstance(recipe_main_data, dict) else 'Not a dict'}") - - # Validate required fields in main data - required_fields = ["name", "description", "version", "globals"] - for field in required_fields: - if field not in recipe_main_data: - raise KeyError(f"Missing required field '{field}' in recipe main data") - self.continue_on_error: bool | None = recipe_main_data.get("continue_on_error") - if self.continue_on_error is not None and not isinstance(self.continue_on_error, bool): - raise ValueError("Top-level continue_on_error must be a boolean") - - #add verification here - - - - - # The rest of the documents are all sequences - sequence_count = 0 - for sequence in recipe_data: - sequence_count += 1 - - logger.debug(f"Processing sequence {sequence_count}: {sequence.get('sequence_name', 'UNNAMED')}") - - if "sequence_name" not in sequence: - logger.error(f"Sequence {sequence_count} missing 'sequence_name' field") - continue - try: - # todo this is failing - self.sequences[sequence["sequence_name"]] = Sequence(sequence_data=sequence) - except Exception as e: - logger.error(f"Failed to create sequence '{sequence.get('sequence_name', 'UNNAMED')}': {e}") - raise - - self.name: str = recipe_main_data["name"] - self.main_sequence: str = recipe_main_data.get("main_sequence", "Main") - if self.main_sequence not in self.sequences: - raise ValueError( - f"Main sequence '{self.main_sequence}' does not exist; " - f"available sequences: {', '.join(self.sequences) or '(none)'}" - ) - self.description: str = recipe_main_data["description"] - self.version: str = recipe_main_data["version"] - - report_mode = recipe_main_data.get("report", "overwrite").lower() - if report_mode == "overwrite": - self.report_overwrite = True - elif report_mode == "append": - self.report_overwrite = False - else: - logger.error(f"'{report_mode}' is not a valid reporting mode. Use 'overwrite' or 'append'.") - raise - - self.report_name_include_serial: bool = bool(recipe_main_data.get("report_name_include_serial", False)) - - self.globals: dict[str, any] = recipe_main_data["globals"] - self.test_package: str = recipe_main_data.get("test_package", None) - if self.test_package and not re.fullmatch( - r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*", self.test_package - ): - raise ValueError( - "test_package must be a valid dotted Python package name " - f"(got {self.test_package!r})" - ) - # self.tags: dict[str, str] = recipe_main_data["tags"] - logger.info(f"Loaded recipe {self.name} version {self.version}.") - logger.debug(f"Recipe has {len(self.sequences)} sequences: {list(self.sequences.keys())}") - - except Exception as e: - logger.error(f"Failed to load recipe from {recipe_file_path}: {e}", exc_info=True) - raise + def _load_definition(self, definition: RecipeDefinition, source_name: str) -> None: + """Build runtime objects only after aggregate validation has succeeded.""" + header = definition.header + logger.info("Loading validated recipe %s.", source_name) + self.definition = definition + self.recipe_file_name = Path(source_name).name + self.sequences = { + sequence.sequence_name: Sequence(sequence) + for sequence in definition.sequences + } + self.name = header.name + self.main_sequence = header.main_sequence + self.description = header.description + self.version = header.version + self.continue_on_error = header.continue_on_error + self.report_overwrite = header.report == "overwrite" + self.report_name_include_serial = header.report_name_include_serial + self.globals = copy.deepcopy(header.globals) + self.test_package = header.test_package + logger.info("Loaded recipe %s version %s.", self.name, self.version) def run(self, runtime: Runtime, sequence_name: str | None = None): """Executes the main sequence of the recipe. @@ -403,9 +348,13 @@ def run(self, runtime: Runtime, sequence_name: str | None = None): # Emit signal so GUI still updates return results - main_step_data = {"steptype": "SequenceStep", "step_name": sequence_name, "sequence": {"type": "internal", "name": sequence_name}, "input_mapping": {}, "output_mapping": {}} - - main_step: Step = Step.build_step(main_step_data) + main_step = ExecutableSequenceStep( + sequence={"type": "internal", "name": sequence_name}, + step_name=sequence_name, + description=f"Run top-level sequence {sequence_name}.", + input_mapping={}, + output_mapping={}, + ) final_result = main_step.run(runtime, {}, stop_event=runtime.stop_event) results: List[StepResult] = runtime.get_results() @@ -438,56 +387,28 @@ def run(self, runtime: Runtime, sequence_name: str | None = None): runtime.send_event("post_run_recipe", results) Runtime.stop_event.set() - - # def parse_q_input(self, q_in): - # while True: - # input_command = q_in.get() - # print(f"RECEIVED SIGNAL FROM GUI: {input_command}") - # event:threading.Event = self.runtime["events"][input_command] - # event.set() - - - - - # @staticmethod - # def run_threaded(recipe_file, sequence_name="Main"): - # q_in = queue.Queue() - # event_queue = queue.SimpleQueue() - # report_queue = queue.SimpleQueue() - # runtime = Runtime(event_queue, report_queue) - # recipe = Recipe(recipe_file) - # runtime.send_event("post_load_recipe", recipe) - # threading.Thread(target=recipe.run, kwargs={"runtime": runtime, "sequence_name": sequence_name}, daemon=True).start() - # # threading.Thread(target=recipe.parse_q_input, args=[q_in], daemon=True).start() - # return event_queue, report_queue, q_in - - class Sequence(): - def __init__(self, sequence_data=None, sequence_file=None): - if sequence_file is not None: - with open(sequence_file, 'r') as file: - sequence_data = yaml.safe_load(file) - elif sequence_data is not None: - sequence_data = sequence_data - else: - raise FileNotFoundError - - self.name = sequence_data["sequence_name"] - self.locals = sequence_data["locals"] - self.parameters = sequence_data["parameters"] - self.outputs = sequence_data["outputs"] + def __init__(self, definition: SequenceDefinition): + if not isinstance(definition, SequenceDefinition): + raise TypeError("Sequence requires a validated recipe_language.Sequence") + self.definition = definition + self.name = definition.sequence_name + self.description = definition.description + self.locals = copy.deepcopy(definition.locals) + self.parameters = copy.deepcopy(definition.parameters) + self.outputs = copy.deepcopy(definition.outputs) self.steps = [] self.teardown_steps = [] # build all contained steps here - for step_data in sequence_data["setup_steps"]: - self.steps.append(Step.build_step(step_data)) + for step_definition in definition.setup_steps: + self.steps.append(Step.build_step(step_definition)) - for step_data in sequence_data["steps"]: - self.steps.append(Step.build_step(step_data)) + for step_definition in definition.steps: + self.steps.append(Step.build_step(step_definition)) - for step_data in sequence_data["teardown_steps"]: - self.teardown_steps.append(Step.build_step(step_data)) + for step_definition in definition.teardown_steps: + self.teardown_steps.append(Step.build_step(step_definition)) def run(self, runtime: Runtime, input: dict, parent_step: uuid.UUID=None): logger.info(f"Starting sequence {self.name}") runtime.send_event("pre_run_sequence", self) @@ -734,33 +655,35 @@ def run_steps(runtime: Runtime, step_list: List[Self], parent_step: uuid.UUID, s return step_results # aggregate_result # single pass or fail type @staticmethod - def build_step(step_data:dict): + def build_step(step_definition: AuthorableStepDefinition): """ - This helper function analyzes the configuration of step_data and adapts what is needed before - creating the step. + This function translates from validated Pydantic step definitions + to the corresponding executable Step objects. Args: - step_data (dict): the dictionary containing the keys from the sequence file + step_definition (AuthorableStepDefinition): the validated step definition Returns: Step: This is a fully configured step object """ - if not isinstance(step_data, dict): - raise TypeError("Step definition must be a dictionary") - step_type = step_data.get("steptype") - if not isinstance(step_type, str): - raise ValueError("Step definition requires a string 'steptype'") - step_class = STEP_TYPE_REGISTRY.get(step_type.casefold()) + if not isinstance(step_definition, AuthorableStepDefinition): + raise TypeError("Step.build_step requires a validated step definition") + step_type = step_definition.steptype + step_class = STEP_TYPE_REGISTRY.get(step_type) if step_class is None: - supported = ", ".join(sorted(cls.__name__ for cls in set(STEP_TYPE_REGISTRY.values()))) + supported = ", ".join(sorted(STEP_TYPE_REGISTRY)) raise ValueError(f"Unknown step type '{step_type}'. Supported step types: {supported}") - constructor_data = dict(step_data) - del constructor_data["steptype"] + constructor_data = step_definition.model_dump( + mode="python", by_alias=True, exclude={"steptype"} + ) new_step: Step = step_class(**constructor_data) - # Check if indexing is to be used, and if so, create IndexingStep to encapsulate the original step - if new_step.check_indexing(): + has_indexed_input = any( + isinstance(value, DirectInputDefinition) and value.indexed + for value in step_definition.input_mapping.values() + ) + if has_indexed_input: # List of keys to keep keys_to_keep = [ @@ -776,16 +699,49 @@ def build_step(step_data:dict): # Import step implementations from steps module -from pypts.steps import IndexedStep, PythonModuleStep, SequenceStep, UserInteractionStep, WaitStep, UserLoadingStep, UserRunMethodStep, UserWriteStep, SerialNumberStep, SSHConnectStep, SSHCloseStep, SSHUploadStep - +from pypts.steps import ( + IndexedStep, + PythonModuleStep as ExecutablePythonModuleStep, + SequenceStep as ExecutableSequenceStep, + UserInteractionStep as ExecutableUserInteractionStep, + WaitStep as ExecutableWaitStep, + UserLoadingStep as ExecutableUserLoadingStep, + UserRunMethodStep as ExecutableUserRunMethodStep, + UserWriteStep as ExecutableUserWriteStep, + SerialNumberStep as ExecutableSerialNumberStep, + SSHConnectStep as ExecutableSSHConnectStep, + SSHCloseStep as ExecutableSSHCloseStep, + SSHUploadStep as ExecutableSSHUploadStep, +) + +# This dictionary maps step type names to their corresponding executable classes, allowing dynamic instantiation based on the step type specified in the recipe. STEP_TYPE_REGISTRY = { - cls.__name__.casefold(): cls for cls in ( - IndexedStep, PythonModuleStep, SequenceStep, UserInteractionStep, - WaitStep, UserLoadingStep, UserRunMethodStep, UserWriteStep, - SerialNumberStep, SSHConnectStep, SSHCloseStep, SSHUploadStep, - ) + "PythonModuleStep": ExecutablePythonModuleStep, + "SequenceStep": ExecutableSequenceStep, + "UserInteractionStep": ExecutableUserInteractionStep, + "WaitStep": ExecutableWaitStep, + "UserLoadingStep": ExecutableUserLoadingStep, + "UserRunMethodStep": ExecutableUserRunMethodStep, + "UserWriteStep": ExecutableUserWriteStep, + "SerialNumberStep": ExecutableSerialNumberStep, + "SSHConnectStep": ExecutableSSHConnectStep, + "SSHCloseStep": ExecutableSSHCloseStep, + "SSHUploadStep": ExecutableSSHUploadStep, } +# Preserve direct concrete runtime construction from this long-standing module. +PythonModuleStep = ExecutablePythonModuleStep +SequenceStep = ExecutableSequenceStep +UserInteractionStep = ExecutableUserInteractionStep +WaitStep = ExecutableWaitStep +UserLoadingStep = ExecutableUserLoadingStep +UserRunMethodStep = ExecutableUserRunMethodStep +UserWriteStep = ExecutableUserWriteStep +SerialNumberStep = ExecutableSerialNumberStep +SSHConnectStep = ExecutableSSHConnectStep +SSHCloseStep = ExecutableSSHCloseStep +SSHUploadStep = ExecutableSSHUploadStep + if __name__ == "__main__": @@ -794,9 +750,6 @@ def build_step(step_data:dict): yaml_dir = os.path.join(os.path.dirname(__file__), 'recipes') yaml_path = os.path.join(yaml_dir, 'simple_recipe.yml') - validate_recipe_filepath(yaml_path) - # give time to print to stdout - time.sleep(0.1) recipe = Recipe(yaml_path) recipe.sequences["Main"].list_steps() diff --git a/spikes/recipe_pydantic/artifacts.py b/src/pypts/recipe_artifacts.py similarity index 76% rename from spikes/recipe_pydantic/artifacts.py rename to src/pypts/recipe_artifacts.py index b10eca3..dbe7a8c 100644 --- a/spikes/recipe_pydantic/artifacts.py +++ b/src/pypts/recipe_artifacts.py @@ -1,4 +1,7 @@ -"""Generate or check the documentation artifacts for recipe language 2.0.""" +# SPDX-FileCopyrightText: 2026 CERN +# +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Generate or check production recipe-language documentation artifacts.""" from __future__ import annotations @@ -8,8 +11,8 @@ from collections.abc import Sequence from pathlib import Path -from .models import Recipe -from .reference import render_reference +from pypts.recipe_language import Recipe +from pypts.recipe_reference import load_schema, render_reference ROOT = Path(__file__).parents[2] DEFAULT_GENERATED_DIR = ROOT / "docs" / "source" / "_generated" @@ -18,7 +21,6 @@ def render_json_schema() -> str: - """Render deterministic JSON Schema from the accepted Pydantic model.""" schema = Recipe.model_json_schema(by_alias=True, mode="validation") schema["$comment"] = ( "SPDX-FileCopyrightText: 2026 CERN ; " @@ -29,22 +31,20 @@ def render_json_schema() -> str: def rendered_artifacts() -> tuple[str, str]: schema_text = render_json_schema() - reference_text = render_reference(json.loads(schema_text)) - return schema_text, reference_text + return schema_text, render_reference(json.loads(schema_text)) def write_artifacts( schema_path: str | Path = DEFAULT_SCHEMA_PATH, reference_path: str | Path = DEFAULT_REFERENCE_PATH, ) -> None: + """Write schema first, then render RST from that exact staged file.""" schema_path = Path(schema_path) reference_path = Path(reference_path) schema_path.parent.mkdir(parents=True, exist_ok=True) reference_path.parent.mkdir(parents=True, exist_ok=True) - schema_path.write_text(render_json_schema(), encoding="utf-8") - schema = json.loads(schema_path.read_text(encoding="utf-8")) - reference_path.write_text(render_reference(schema), encoding="utf-8") + reference_path.write_text(render_reference(load_schema(schema_path)), encoding="utf-8") def check_artifacts( @@ -62,16 +62,12 @@ def check_artifacts( return current == expected -def _parser() -> argparse.ArgumentParser: +def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--check", action="store_true", help="fail if either artifact is stale") + parser.add_argument("--check", action="store_true") parser.add_argument("--schema", type=Path, default=DEFAULT_SCHEMA_PATH) parser.add_argument("--reference", type=Path, default=DEFAULT_REFERENCE_PATH) - return parser - - -def main(argv: Sequence[str] | None = None) -> int: - arguments = _parser().parse_args(argv) + arguments = parser.parse_args(argv) if arguments.check: if check_artifacts(arguments.schema, arguments.reference): return 0 @@ -82,8 +78,6 @@ def main(argv: Sequence[str] | None = None) -> int: except OSError as error: print(f"Could not write recipe documentation artifacts: {error}", file=sys.stderr) return 2 - print(f"Wrote {arguments.schema}") - print(f"Wrote {arguments.reference}") return 0 diff --git a/src/pypts/recipe_language.py b/src/pypts/recipe_language.py index a418b85..6544289 100644 --- a/src/pypts/recipe_language.py +++ b/src/pypts/recipe_language.py @@ -1,701 +1,389 @@ # SPDX-FileCopyrightText: 2026 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -"""The framework-independent contract for the recipe YAML language. +"""Authoritative Pydantic model for the candidate recipe language. -This module deliberately accepts already-loaded Python dictionaries. Loading -YAML, retaining source locations, and constructing runtime steps are separate -concerns which will be introduced by the parser and integration work. +Field declarations intentionally own types, defaults, descriptions, examples, +serialization behavior, and JSON Schema. There is no parallel field registry. """ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Any, Iterable, Mapping +from typing import Annotated, Any, Literal, get_args +from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic_core import PydanticCustomError -CANONICAL_RECIPE_VERSION = "1.0.0" -NO_DEFAULT = object() +def described(description: str, *, example: Any = None, **kwargs: Any) -> Any: + """Small spelling helper; returned metadata still lives on each Field.""" + examples = None if example is None else [example] + return Field(description=description, examples=examples, **kwargs) -@dataclass(frozen=True) -class SourcePosition: - """A one-based source position with a zero-based character offset.""" - line: int - column: int - offset: int +class RecipeModel(BaseModel): + """Strict, immutable base for all authorable structures.""" + model_config = ConfigDict( + extra="forbid", + frozen=True, + strict=True, + populate_by_name=True, + validate_default=True, + ) -@dataclass(frozen=True) -class SourceSpan: - """Half-open source range.""" - start: SourcePosition - end: SourcePosition +class DirectInput(RecipeModel): + """Provides a literal value.""" + type: Literal["direct"] = described("Input source type.", example="direct") + value: Any = described("Literal input value.", example=1) + indexed: bool = described( + "Expand a list into indexed steps.", example=False, default=False, + exclude_if=lambda value: not value, + ) -@dataclass(frozen=True) -class Diagnostic: - """A language-contract finding for an already-loaded recipe document.""" + # docs:indexed-direct-start + @model_validator(mode="after") + def indexed_values_are_lists(self) -> DirectInput: + if self.indexed and not isinstance(self.value, list): + raise PydanticCustomError( + "invalid_indexed_input", "Indexed direct input value must be a list." + ) + return self + # docs:indexed-direct-end - code: str - message: str - path: tuple[str | int, ...] = () - severity: str = "error" - source_name: str | None = None - span: SourceSpan | None = None +class LocalInput(RecipeModel): + """Reads a sequence-local variable.""" -@dataclass(frozen=True) -class ValidationResult: - diagnostics: tuple[Diagnostic, ...] = () + type: Literal["local"] = described("Input source type.", example="local") + local_name: str = described("Local variable name.", example="local_value") - @property - def errors(self) -> tuple[Diagnostic, ...]: - return tuple(item for item in self.diagnostics if item.severity == "error") - @property - def warnings(self) -> tuple[Diagnostic, ...]: - return tuple(item for item in self.diagnostics if item.severity == "warning") +class GlobalInput(RecipeModel): + """Reads a recipe-global variable.""" - @property - def is_valid(self) -> bool: - return not self.errors + type: Literal["global"] = described("Input source type.", example="global") + global_name: str = described("Global variable name.", example="global_value") -@dataclass(frozen=True) -class FieldSpec: - name: str - value_type: type | tuple[type, ...] | None = None - required: bool = False - description: str = "" - default: Any = NO_DEFAULT - choices: tuple[Any, ...] = () - legacy: bool = False - choice_diagnostic: str = "invalid-field-value" +class MethodInput(RecipeModel): + """Resolves a method reference for the step.""" - @property - def has_default(self) -> bool: - return self.default is not NO_DEFAULT + type: Literal["method"] = described("Input source type.", example="method") + value: Any = described("Method reference.", example="helper") -@dataclass(frozen=True) -class StructureSpec: - """Declarative fields for a recipe document or nested structure.""" - - name: str - description: str - fields: tuple[FieldSpec, ...] - - @property - def fields_by_name(self) -> dict[str, FieldSpec]: - return {field.name: field for field in self.fields} - - -@dataclass(frozen=True) -class MappingSpec: - """Declarative contract for one input or output mapping variant.""" - - name: str - description: str - fields: tuple[FieldSpec, ...] - example: Mapping[str, Any] - verdict: bool = False - - @property - def fields_by_name(self) -> dict[str, FieldSpec]: - return {field.name: field for field in self.fields} - - -@dataclass(frozen=True) -class ConstraintSpec: - """Human-readable metadata for a semantic language constraint.""" - - code: str - scope: str - description: str - diagnostic_codes: tuple[str, ...] = () - - -@dataclass(frozen=True) -class StepSpec: - """Declarative contract for one registered recipe step type.""" - - name: str - fields: tuple[FieldSpec, ...] = () - required_inputs: tuple[str, ...] = () - example: Mapping[str, Any] = field(default_factory=dict) - description: str = "" - source_allowed: bool = True - - @property - def fields_by_name(self) -> dict[str, FieldSpec]: - return {field.name: field for field in self.fields} - - -COMMON_STEP_FIELDS = ( - FieldSpec("steptype", str, required=True, description="Registered step type."), - FieldSpec("step_name", str, required=True, description="Human-readable step name."), - FieldSpec("description", str, required=True, description="Purpose of the step."), - FieldSpec("id", str, description="Optional stable step identifier."), - FieldSpec("skip", bool, description="Skip execution.", default=False), - FieldSpec("critical", bool, description="Stop on error when policy permits continuation.", default=False), - FieldSpec("continue_on_error", bool, description="Per-step error policy.", default=False), - FieldSpec("input_mapping", dict, description="Named input sources.", default={}), - FieldSpec("output_mapping", dict, description="Named verdicts and destinations.", default={}), -) - -COMMON_STEP_SPEC = StructureSpec( - "step", - "Fields accepted by every authorable recipe step.", - COMMON_STEP_FIELDS, -) - - -def _step_spec( - name: str, - *fields: FieldSpec, - required_inputs: tuple[str, ...] = (), - example: Mapping[str, Any], - description: str, - source_allowed: bool = True, -) -> StepSpec: - return StepSpec(name, COMMON_STEP_FIELDS + fields, required_inputs, example, description, source_allowed) - - -STEP_SPECS = ( - _step_spec( - "PythonModuleStep", - FieldSpec( - "action_type", str, required=True, - description="Operation performed on the Python module.", - choices=("method", "read_attribute", "write_attribute"), - choice_diagnostic="invalid-action-type", - ), - FieldSpec("module", str, required=True, description="Python module path."), - FieldSpec("method_name", str, description="Method name required by method actions."), - example={"steptype": "PythonModuleStep", "step_name": "Run test", "description": "Run a Python test method.", "action_type": "method", "module": "tests.py", "method_name": "run", "input_mapping": {}, "output_mapping": {}}, - description="Calls a method or reads/writes an attribute in a Python module.", - ), - _step_spec( - "SequenceStep", - FieldSpec("sequence", dict, required=True, description="Internal sequence reference."), - example={"steptype": "SequenceStep", "step_name": "Run calibration", "description": "Run an internal sequence.", "sequence": {"type": "internal", "name": "Calibration"}, "input_mapping": {}, "output_mapping": {}}, - description="Runs another sequence as a step.", - ), - _step_spec( - "UserInteractionStep", - example={"steptype": "UserInteractionStep", "step_name": "Confirm", "description": "Ask the operator to confirm.", "input_mapping": {"message": {"type": "direct", "value": "Continue?"}}, "output_mapping": {"output": {"type": "passfail"}}}, - description="Displays an operator interaction prompt.", - ), - _step_spec( - "WaitStep", - required_inputs=("wait_time",), - example={"steptype": "WaitStep", "step_name": "Stabilize", "description": "Wait for hardware stabilization.", "input_mapping": {"wait_time": {"type": "direct", "value": 1}}, "output_mapping": {}}, - description="Waits for a non-negative duration in seconds.", - ), - _step_spec( - "UserLoadingStep", - FieldSpec("file_save_location", dict, description="Local or global destination for the selected file."), - example={"steptype": "UserLoadingStep", "step_name": "Load configuration", "description": "Ask the operator for a file.", "input_mapping": {"message": {"type": "direct", "value": "Choose a file"}}, "output_mapping": {"output": {"type": "passfail"}}}, - description="Prompts the operator to select a file.", - ), - _step_spec( - "UserRunMethodStep", - FieldSpec("trigger_response", (str, list, dict), description="Operator response that triggers execution."), - FieldSpec("action_type", str, description="Optional Python action type."), - FieldSpec("module", str, description="Optional Python module path."), - FieldSpec("method_name", str, description="Optional Python method name."), - example={"steptype": "UserRunMethodStep", "step_name": "Run calibration", "description": "Run on operator confirmation.", "trigger_response": "run", "action_type": "method", "module": "tests.py", "method_name": "calibrate", "input_mapping": {}, "output_mapping": {"output": {"type": "passfail"}}}, - description="Optionally runs a Python method after an operator response.", - ), - _step_spec( - "UserWriteStep", - example={"steptype": "UserWriteStep", "step_name": "Enter value", "description": "Ask the operator for a value.", "input_mapping": {"message": {"type": "direct", "value": "Enter value"}}, "output_mapping": {"output": {"type": "local", "local_name": "value"}}}, - description="Writes an operator-provided value to a configured destination.", - ), - _step_spec( - "SerialNumberStep", - example={"steptype": "SerialNumberStep", "step_name": "Scan serial number", "description": "Capture the device serial number.", "input_mapping": {}, "output_mapping": {}}, - description="Captures the device serial number.", - ), - _step_spec( - "SSHConnectStep", - example={"steptype": "SSHConnectStep", "step_name": "Connect", "description": "Open the SSH connection."}, - description="Opens the SSH client stored in recipe globals.", - ), - _step_spec( - "SSHCloseStep", - example={"steptype": "SSHCloseStep", "step_name": "Disconnect", "description": "Close the SSH connection."}, - description="Closes the SSH client stored in recipe globals.", - ), - _step_spec( - "SSHUploadStep", - FieldSpec("files", list, required=True, description="Local and remote file pairs to upload."), - FieldSpec("permissions", (int, str), description="Optional remote permissions."), - FieldSpec("skip_if_sha256_match", bool, description="Skip files whose remote checksum matches.", default=False), - FieldSpec("local_package", str, description="Optional package containing local resources."), - example={"steptype": "SSHUploadStep", "step_name": "Deploy", "description": "Upload a file to the target.", "files": [{"local": "bin/tool", "remote": "/tmp/tool"}], "output_mapping": {"passed": {"type": "passfail"}}}, - description="Uploads files through an SSH connection.", - ), - _step_spec( - "IndexedStep", - example={"steptype": "IndexedStep", "step_name": "Indexed operation", "description": "Reserved runtime wrapper step.", "input_mapping": {}, "output_mapping": {}}, - description="Reserved for the runtime's automatic indexed-step wrapper.", - source_allowed=False, - ), -) - -STEP_SPECS_BY_NAME = {spec.name.casefold(): spec for spec in STEP_SPECS} - -HEADER_FIELDS = ( - FieldSpec("name", str, required=True, description="Human-readable recipe name."), - FieldSpec("version", str, required=True, description="Version of this recipe."), - FieldSpec( - "recipe_version", str, required=True, - description="Version of the recipe language contract.", - choices=(CANONICAL_RECIPE_VERSION,), - choice_diagnostic="unsupported-recipe-version", - ), - FieldSpec("description", str, required=True, description="Purpose of the recipe."), - FieldSpec("main_sequence", str, required=True, description="Sequence where execution begins."), - FieldSpec("globals", dict, required=True, description="Recipe-wide variables."), - FieldSpec("continue_on_error", bool, description="Recipe-wide error policy.", default=None), - FieldSpec( - "report", str, description="Report file mode.", default="overwrite", - choices=("overwrite", "append"), choice_diagnostic="invalid-report-mode", - ), - FieldSpec( - "report_name_include_serial", bool, - description="Include the serial number in the report name.", default=False, - ), - FieldSpec("test_package", str, description="Package containing recipe test modules.", default=None), -) -HEADER_SPEC = StructureSpec( - "recipe header", - "The first YAML document; identifies the recipe and its entry sequence.", - HEADER_FIELDS, -) - -SEQUENCE_FIELDS = ( - FieldSpec("sequence_name", str, required=True, description="Unique sequence name."), - FieldSpec("description", str, required=True, description="Purpose of the sequence."), - FieldSpec("parameters", dict, required=True, description="Reserved sequence input metadata."), - FieldSpec("outputs", dict, required=True, description="Reserved sequence output metadata."), - FieldSpec("locals", dict, required=True, description="Variables local to the sequence."), - FieldSpec("setup_steps", list, required=True, description="Steps run before the main steps."), - FieldSpec("steps", list, required=True, description="Ordered main steps."), - FieldSpec("teardown_steps", list, required=True, description="Steps run during teardown."), - FieldSpec( - "serial_number", (str, int), - description="Runtime-ignored legacy sequence metadata.", legacy=True, - ), -) -SEQUENCE_SPEC = StructureSpec( - "sequence", - "Each YAML document after the header defines one named sequence.", - SEQUENCE_FIELDS, -) - -SEQUENCE_REFERENCE_SPEC = StructureSpec( - "internal sequence reference", - "Reference used by SequenceStep.", - ( - FieldSpec("type", str, required=True, description="Reference kind.", choices=("internal",)), - FieldSpec("name", str, required=True, description="Target sequence name."), - ), -) - -FILE_SAVE_LOCATION_SPEC = StructureSpec( - "file save location", - "Destination used by UserLoadingStep.", - ( - FieldSpec("type", str, required=True, description="Variable scope.", choices=("local", "global")), - FieldSpec("variable", str, required=True, description="Destination variable name."), - ), -) - -INPUT_MAPPING_SPECS = ( - MappingSpec( - "direct", "Provides a literal value.", - ( - FieldSpec("type", str, description="Input source type.", default="direct", choices=("direct",)), - FieldSpec("value", required=True, description="Literal input value."), - FieldSpec("indexed", bool, description="Expand a list into indexed steps.", default=False), - ), - {"type": "direct", "value": 1}, - ), - MappingSpec( - "local", "Reads a sequence-local variable.", - ( - FieldSpec("type", str, required=True, description="Input source type.", choices=("local",)), - FieldSpec("local_name", str, required=True, description="Local variable name."), - FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), - ), - {"type": "local", "local_name": "local_value"}, - ), - MappingSpec( - "global", "Reads a recipe-global variable.", - ( - FieldSpec("type", str, required=True, description="Input source type.", choices=("global",)), - FieldSpec("global_name", str, required=True, description="Global variable name."), - FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), - ), - {"type": "global", "global_name": "global_value"}, - ), - MappingSpec( - "method", "Resolves a method reference for the step.", - ( - FieldSpec("type", str, required=True, description="Input source type.", choices=("method",)), - FieldSpec("value", required=True, description="Method reference."), - FieldSpec("indexed", bool, description="Compatibility field; only false is accepted.", default=False, legacy=True), - ), - {"type": "method", "value": "helper"}, - ), -) -INPUT_MAPPING_SPECS_BY_NAME = {spec.name: spec for spec in INPUT_MAPPING_SPECS} - -OUTPUT_MAPPING_SPECS = ( - MappingSpec( - "passfail", "Interprets the output as a pass/fail verdict.", - (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("passfail",)),), - {"type": "passfail"}, verdict=True, - ), - MappingSpec( - "equals", "Passes when the output equals the configured value.", - ( - FieldSpec("type", str, required=True, description="Output mapping type.", choices=("equals",)), - FieldSpec("value", required=True, description="Expected value."), - ), - {"type": "equals", "value": 3}, verdict=True, - ), - MappingSpec( - "range", "Passes when the output is within an inclusive range.", - ( - FieldSpec("type", str, required=True, description="Output mapping type.", choices=("range",)), - FieldSpec("min", required=True, description="Minimum accepted value."), - FieldSpec("max", required=True, description="Maximum accepted value."), - ), - {"type": "range", "min": 1, "max": 4}, verdict=True, - ), - MappingSpec( - "passthrough", "Uses the nested result without adding a verdict.", - (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("passthrough",)),), - {"type": "passthrough"}, verdict=True, - ), - MappingSpec( - "local", "Stores the output in a sequence-local variable.", - ( - FieldSpec("type", str, required=True, description="Output mapping type.", choices=("local",)), - FieldSpec("local_name", str, required=True, description="Local destination variable."), - ), - {"type": "local", "local_name": "saved"}, - ), - MappingSpec( - "global", "Stores the output in a recipe-global variable.", - ( - FieldSpec("type", str, required=True, description="Output mapping type.", choices=("global",)), - FieldSpec("global_name", str, required=True, description="Global destination variable."), - ), - {"type": "global", "global_name": "saved"}, - ), - MappingSpec( - "image", "Publishes an image output for presentation.", - (FieldSpec("type", str, required=True, description="Output mapping type.", choices=("image",)),), - {"type": "image"}, - ), -) -OUTPUT_MAPPING_SPECS_BY_NAME = {spec.name: spec for spec in OUTPUT_MAPPING_SPECS} - -CONSTRAINT_SPECS = ( - ConstraintSpec( - "safe-source", "parser", - "Sources must be readable text and safe, non-recursive YAML without duplicate keys.", - ("invalid-source", "file-read-error", "yaml-syntax-error", "yaml-construction-error", "unsafe-yaml", "recursive-alias", "duplicate-key"), - ), - ConstraintSpec( - "declared-fields", "documents and steps", - "Only declared fields, field types, and allowed values are accepted.", - ("missing-field", "invalid-field-type", "invalid-field-value", "unknown-field", "unsupported-recipe-version", "invalid-report-mode", "invalid-action-type"), - ), - ConstraintSpec( - "mapping-shape", "input and output mappings", - "Mapping variants accept only their declared fields and require their declared values.", - ("invalid-input-mapping", "unknown-input-source", "unknown-input-field", "invalid-input-field-type", "missing-input-source-value", "invalid-output-mapping", "unknown-output-type", "unknown-output-field", "invalid-output-field-type", "missing-output-field"), - ), - ConstraintSpec( - "recipe-documents", "recipe", - "A recipe is safe multi-document YAML with one header followed by at least one sequence.", - ("empty-recipe", "invalid-header", "invalid-sequence"), - ), - ConstraintSpec( - "registered-step", "step", "Every step is a mapping with a registered step type.", - ("invalid-step", "unknown-step-type"), - ), - ConstraintSpec( - "unique-sequences", "sequence", - "Sequence names are unique and main_sequence names an existing sequence.", - ("duplicate-sequence", "unknown-main-sequence"), - ), - ConstraintSpec( - "internal-sequence-reference", "SequenceStep", - "Only internal references are accepted and the named target sequence must exist.", - ("invalid-sequence-reference", "unknown-sequence-reference"), - ), - ConstraintSpec( - "method-action-name", "PythonModuleStep", "A method action requires method_name.", - ("missing-method-name",), - ), - ConstraintSpec( - "file-save-location", "UserLoadingStep", "A file destination names a local or global variable.", - ("invalid-file-save-location",), - ), - ConstraintSpec( - "indexed-inputs", "input mapping", - "Indexed inputs are direct lists and all indexed lists on a step have equal length.", - ("invalid-indexed-flag", "invalid-indexed-input", "unequal-indexed-inputs"), - ), - ConstraintSpec( - "passthrough-verdict", "output mapping", "passthrough must be the only verdict mapping on its step.", - ("mixed-passthrough",), - ), - ConstraintSpec( - "required-step-inputs", "step", "Step-specific required input names must be present.", - ("missing-required-input", "missing-input-mapping"), - ), - ConstraintSpec( - "ssh-context", "SSH steps", - "SSH steps require connection globals; setup connections require teardown closure, and uploads require an earlier connection.", - ("missing-ssh-global", "missing-ssh-credential", "missing-ssh-connect", "missing-ssh-close"), - ), - ConstraintSpec( - "internal-step", "step", "IndexedStep is runtime-generated and cannot be authored in recipe YAML.", - ("internal-step-type",), - ), - ConstraintSpec( - "legacy-sequence-field", "sequence", "serial_number is accepted with a warning and omitted from the typed model.", - ("legacy-sequence-field",), - ), - ConstraintSpec( - "canonical-spelling", "step and input mapping", - "Noncanonical step casing and omitted direct input types are normalized with warnings.", - ("noncanonical-step-type", "implicit-direct-input"), - ), -) -CONSTRAINT_SPECS_BY_CODE = {spec.code: spec for spec in CONSTRAINT_SPECS} -DOCUMENTED_DIAGNOSTIC_CODES = frozenset( - code for spec in CONSTRAINT_SPECS for code in spec.diagnostic_codes -) - - -def canonical_step_type(step_type: str) -> str | None: - spec = STEP_SPECS_BY_NAME.get(step_type.casefold()) if isinstance(step_type, str) else None - return spec.name if spec else None - - -def _type_name(value_type: type | tuple[type, ...]) -> str: - values = value_type if isinstance(value_type, tuple) else (value_type,) - return " or ".join(value.__name__ for value in values) - - -def _matches_type(value: Any, value_type: type | tuple[type, ...]) -> bool: - if value_type is bool: - return type(value) is bool - return isinstance(value, value_type) - - -def _check_fields(value: Mapping[str, Any], fields: Iterable[FieldSpec], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: - specs = {spec.name: spec for spec in fields} - for name, spec in specs.items(): - if spec.required and name not in value: - diagnostics.append(Diagnostic("missing-field", f"Missing required field '{name}'.", path + (name,))) - elif name in value and spec.value_type is not None and not _matches_type(value[name], spec.value_type): - diagnostics.append(Diagnostic("invalid-field-type", f"Field '{name}' must be {_type_name(spec.value_type)}.", path + (name,))) - elif name in value and spec.choices and value[name] not in spec.choices: - choices = ", ".join(repr(choice) for choice in spec.choices) - diagnostics.append(Diagnostic( - spec.choice_diagnostic, - f"Field '{name}' must be one of: {choices}.", - path + (name,), - )) - for name in value: - if name not in specs: - diagnostics.append(Diagnostic("unknown-field", f"Unknown field '{name}'.", path + (name,))) - - -def _validate_input_mappings(mapping: Mapping[str, Any], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: - indexed_lengths: list[int] = [] - for name, config in mapping.items(): - item_path = path + (name,) - if not isinstance(config, Mapping): - diagnostics.append(Diagnostic("invalid-input-mapping", "Input mapping must be a dictionary.", item_path)) - continue - source = config.get("type", "direct") - if source not in INPUT_MAPPING_SPECS_BY_NAME: - diagnostics.append(Diagnostic("unknown-input-source", f"Unknown input source '{source}'.", item_path + ("type",))) - continue - spec = INPUT_MAPPING_SPECS_BY_NAME[source] - allowed = spec.fields_by_name - for field_name in config: - if field_name not in allowed: - diagnostics.append(Diagnostic("unknown-input-field", f"Unknown field '{field_name}' for input source '{source}'.", item_path + (field_name,))) - for field_name, field_spec in allowed.items(): - if field_name in {"type", "indexed"} or field_name not in config or field_spec.value_type is None: - continue - if not _matches_type(config[field_name], field_spec.value_type): - diagnostics.append(Diagnostic( - "invalid-input-field-type", - f"Input field '{field_name}' must be {_type_name(field_spec.value_type)}.", - item_path + (field_name,), - )) - for field_spec in spec.fields: - if field_spec.required and field_spec.name not in config: - diagnostics.append(Diagnostic("missing-input-source-value", f"Input source '{source}' requires '{field_spec.name}'.", item_path)) - if "indexed" in config and type(config["indexed"]) is not bool: - diagnostics.append(Diagnostic("invalid-indexed-flag", "'indexed' must be boolean.", item_path + ("indexed",))) - if config.get("indexed"): - if source != "direct" or not isinstance(config.get("value"), list): - diagnostics.append(Diagnostic("invalid-indexed-input", "Indexed inputs must be direct lists.", item_path)) - else: - indexed_lengths.append(len(config["value"])) - if len(set(indexed_lengths)) > 1: - diagnostics.append(Diagnostic("unequal-indexed-inputs", "Indexed input lists must have equal lengths.", path)) - - -def _validate_output_mappings(mapping: Mapping[str, Any], path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> None: - verdicts: list[str] = [] - for name, config in mapping.items(): - item_path = path + (name,) - if not isinstance(config, Mapping) or not isinstance(config.get("type"), str): - diagnostics.append(Diagnostic("invalid-output-mapping", "Output mapping requires a string 'type'.", item_path)) - continue - kind = config["type"] - if kind not in OUTPUT_MAPPING_SPECS_BY_NAME: - diagnostics.append(Diagnostic("unknown-output-type", f"Unknown output type '{kind}'.", item_path + ("type",))) - continue - spec = OUTPUT_MAPPING_SPECS_BY_NAME[kind] - allowed = spec.fields_by_name - for field_name in config: - if field_name not in allowed: - diagnostics.append(Diagnostic("unknown-output-field", f"Unknown field '{field_name}' for output type '{kind}'.", item_path + (field_name,))) - for field_name, field_spec in allowed.items(): - if field_name == "type" or field_name not in config or field_spec.value_type is None: - continue - if not _matches_type(config[field_name], field_spec.value_type): - diagnostics.append(Diagnostic( - "invalid-output-field-type", - f"Output field '{field_name}' must be {_type_name(field_spec.value_type)}.", - item_path + (field_name,), - )) - if spec.verdict: - verdicts.append(kind) - for field_spec in spec.fields: - if field_spec.required and field_spec.name not in config: - diagnostics.append(Diagnostic("missing-output-field", f"Output type '{kind}' requires '{field_spec.name}'.", item_path)) - if "passthrough" in verdicts and len(verdicts) != 1: - diagnostics.append(Diagnostic("mixed-passthrough", "'passthrough' must be the sole verdict mapping.", path)) - - -def _validate_step(step: Any, path: tuple[str | int, ...], diagnostics: list[Diagnostic]) -> str | None: - if not isinstance(step, Mapping): - diagnostics.append(Diagnostic("invalid-step", "Step must be a dictionary.", path)) - return None - step_type = step.get("steptype") - canonical_name = canonical_step_type(step_type) - if canonical_name is None: - diagnostics.append(Diagnostic("unknown-step-type", f"Unknown step type '{step_type}'.", path + ("steptype",))) - return None - spec = STEP_SPECS_BY_NAME[canonical_name.casefold()] - if not spec.source_allowed: - diagnostics.append(Diagnostic("internal-step-type", f"{canonical_name} is created by the runtime and cannot be written in a recipe.", path + ("steptype",))) - return canonical_name - _check_fields(step, spec.fields, path, diagnostics) - if "input_mapping" in step and isinstance(step["input_mapping"], Mapping): - _validate_input_mappings(step["input_mapping"], path + ("input_mapping",), diagnostics) - for required in spec.required_inputs: - if required not in step["input_mapping"]: - diagnostics.append(Diagnostic("missing-required-input", f"{canonical_name} requires input '{required}'.", path + ("input_mapping", required))) - elif spec.required_inputs: - diagnostics.append(Diagnostic("missing-input-mapping", f"{canonical_name} requires an input_mapping.", path + ("input_mapping",))) - if "output_mapping" in step and isinstance(step["output_mapping"], Mapping): - _validate_output_mappings(step["output_mapping"], path + ("output_mapping",), diagnostics) - if canonical_name == "PythonModuleStep": - if step.get("action_type") == "method" and not step.get("method_name"): - diagnostics.append(Diagnostic("missing-method-name", "PythonModuleStep method action requires method_name.", path + ("method_name",))) - if canonical_name == "SequenceStep": - sequence = step.get("sequence") - if not isinstance(sequence, Mapping) or sequence.get("type") != "internal" or not isinstance(sequence.get("name"), str): - diagnostics.append(Diagnostic("invalid-sequence-reference", "SequenceStep requires sequence.type 'internal' and string sequence.name.", path + ("sequence",))) - if canonical_name == "UserLoadingStep" and "file_save_location" in step: - location = step["file_save_location"] - if not isinstance(location, Mapping) or location.get("type") not in {"local", "global"} or not isinstance(location.get("variable"), str): - diagnostics.append(Diagnostic("invalid-file-save-location", "file_save_location requires type local/global and string variable.", path + ("file_save_location",))) - return canonical_name - - -def validate_recipe_documents(documents: Iterable[Any]) -> ValidationResult: - """Validate already-loaded documents against the canonical language model. - - The function has no YAML dependency and intentionally performs no runtime - imports or execution. A future parser will supply source spans and YAML - loading before calling this contract validator. - """ - docs = list(documents) - diagnostics: list[Diagnostic] = [] - if not docs: - return ValidationResult((Diagnostic("empty-recipe", "A recipe requires a header and at least one sequence."),)) - header = docs[0] - if not isinstance(header, Mapping): - return ValidationResult((Diagnostic("invalid-header", "The first document must be the recipe header.", (0,)),)) - _check_fields(header, HEADER_FIELDS, (0,), diagnostics) - sequences: dict[str, Mapping[str, Any]] = {} - sequence_step_types: dict[str, dict[str, list[str]]] = {} - for doc_index, sequence in enumerate(docs[1:], start=1): - path = (doc_index,) - if not isinstance(sequence, Mapping): - diagnostics.append(Diagnostic("invalid-sequence", "Sequence document must be a dictionary.", path)) - continue - _check_fields(sequence, SEQUENCE_FIELDS, path, diagnostics) - name = sequence.get("sequence_name") - if isinstance(name, str): - if name in sequences: - diagnostics.append(Diagnostic("duplicate-sequence", f"Duplicate sequence '{name}'.", path + ("sequence_name",))) - sequences[name] = sequence - if "serial_number" in sequence: - diagnostics.append(Diagnostic("legacy-sequence-field", "'serial_number' is runtime-ignored legacy metadata.", path + ("serial_number",), "warning")) - sections: dict[str, list[str]] = {} - for section in ("setup_steps", "steps", "teardown_steps"): - values = sequence.get(section, []) - section_types: list[str] = [] - if isinstance(values, list): - for index, step in enumerate(values): - step_type = _validate_step(step, path + (section, index), diagnostics) - if step_type: - section_types.append(step_type) - sections[section] = section_types - if isinstance(name, str): - sequence_step_types[name] = sections - if isinstance(header, Mapping) and isinstance(header.get("main_sequence"), str) and header["main_sequence"] not in sequences: - diagnostics.append(Diagnostic("unknown-main-sequence", f"Main sequence '{header['main_sequence']}' does not exist.", (0, "main_sequence"))) - for sequence_name, sequence in sequences.items(): - for section in ("setup_steps", "steps", "teardown_steps"): - for index, step in enumerate(sequence.get(section, [])): - if isinstance(step, Mapping) and canonical_step_type(step.get("steptype")) == "SequenceStep": - target = step.get("sequence", {}).get("name") if isinstance(step.get("sequence"), Mapping) else None - if isinstance(target, str) and target not in sequences: - diagnostics.append(Diagnostic("unknown-sequence-reference", f"Sequence '{sequence_name}' references unknown sequence '{target}'.", (sequence_name, section, index, "sequence", "name"))) - kinds = sequence_step_types.get(sequence_name, {}) - all_types = [kind for values in kinds.values() for kind in values] - if any(kind.startswith("SSH") for kind in all_types): - globals_data = header.get("globals", {}) if isinstance(header, Mapping) else {} - for required in ("ssh_client", "host", "user", "port"): - if required not in globals_data: - diagnostics.append(Diagnostic("missing-ssh-global", f"SSH step requires global '{required}'.", (0, "globals", required))) - if "password" not in globals_data and "private_key" not in globals_data: - diagnostics.append(Diagnostic("missing-ssh-credential", "SSH steps require password or private_key global.", (0, "globals"))) - if "SSHUploadStep" in kinds.get("steps", []) and "SSHConnectStep" not in kinds.get("setup_steps", []): - diagnostics.append(Diagnostic("missing-ssh-connect", f"Sequence '{sequence_name}' uses SSHUploadStep without setup SSHConnectStep.", (sequence_name, "steps"))) - if "SSHConnectStep" in kinds.get("setup_steps", []) and "SSHCloseStep" not in kinds.get("teardown_steps", []): - diagnostics.append(Diagnostic("missing-ssh-close", f"Sequence '{sequence_name}' setup SSHConnectStep requires teardown SSHCloseStep.", (sequence_name, "teardown_steps"))) - return ValidationResult(tuple(diagnostics)) +type InputMapping = Annotated[ + DirectInput | LocalInput | GlobalInput | MethodInput, + Field(discriminator="type"), +] + + +class PassFailOutput(RecipeModel): + """Interprets the output as a pass/fail verdict.""" + + type: Literal["passfail"] = described("Output mapping type.", example="passfail") + + +class EqualsOutput(RecipeModel): + """Passes when the output equals the configured value.""" + + type: Literal["equals"] = described("Output mapping type.", example="equals") + value: Any = described("Expected value.", example=3) + + +class RangeOutput(RecipeModel): + """Passes when the output is within an inclusive range.""" + + type: Literal["range"] = described("Output mapping type.", example="range") + minimum: Any = described("Minimum accepted value.", example=1, alias="min") + maximum: Any = described("Maximum accepted value.", example=4, alias="max") + + +class PassthroughOutput(RecipeModel): + """Uses the nested result without adding a verdict.""" + + type: Literal["passthrough"] = described("Output mapping type.", example="passthrough") + + +class LocalOutput(RecipeModel): + """Stores the output in a sequence-local variable.""" + + type: Literal["local"] = described("Output mapping type.", example="local") + local_name: str = described("Local destination variable.", example="saved") + + +class GlobalOutput(RecipeModel): + """Stores the output in a recipe-global variable.""" + + type: Literal["global"] = described("Output mapping type.", example="global") + global_name: str = described("Global destination variable.", example="saved") + + +class ImageOutput(RecipeModel): + """Publishes an image output for presentation.""" + + type: Literal["image"] = described("Output mapping type.", example="image") + + +type OutputMapping = Annotated[ + PassFailOutput + | EqualsOutput + | RangeOutput + | PassthroughOutput + | LocalOutput + | GlobalOutput + | ImageOutput, + Field(discriminator="type"), +] + + +class InternalSequenceReference(RecipeModel): + """Reference to another sequence in this recipe.""" + + type: Literal["internal"] = described("Reference kind.", example="internal") + name: str = described("Target sequence name.", example="Calibration") + + +class FileDestination(RecipeModel): + """Destination used by a file-loading step.""" + + type: Literal["local", "global"] = described("Variable scope.", example="local") + variable: str = described("Destination variable name.", example="selected_file") + + +class UploadFile(RecipeModel): + """One local-to-remote SSH upload pair.""" + + local: str = described("Local file or package resource.", example="bin/tool") + remote: str = described("Remote destination path.", example="/tmp/tool") + + +class CommonStep(RecipeModel): + """Fields shared by every authorable step.""" + + step_name: str = described("Human-readable step name.", example="Run test") + description: str = described("Purpose of the step.", example="Run a test operation.") + id: str | None = described("Optional stable step identifier.", example="test-1", default=None) + skip: bool = described("Skip execution.", example=False, default=False) + critical: bool = described( + "Stop on error when policy permits continuation.", example=False, default=False + ) + continue_on_error: bool = described("Per-step error policy.", example=False, default=False) + input_mapping: dict[str, InputMapping] = described( + "Named input sources.", example={}, default_factory=dict + ) + output_mapping: dict[str, OutputMapping] = described( + "Named verdicts and destinations.", example={}, default_factory=dict + ) + + +class PythonModuleStep(CommonStep): + """Calls a method or reads/writes an attribute in a Python module.""" + + steptype: Literal["PythonModuleStep"] = described( + "Canonical registered step type.", example="PythonModuleStep" + ) + action_type: Literal["method", "read_attribute", "write_attribute"] = described( + "Operation performed on the Python module.", example="method" + ) + module: str = described("Python module path.", example="tests.py") + method_name: str | None = described("Method name for method actions.", example="run", default=None) + + # docs:method-name-start + @model_validator(mode="after") + def method_actions_have_names(self) -> PythonModuleStep: + if self.action_type == "method" and not self.method_name: + raise PydanticCustomError( + "missing_method_name", "Method actions require method_name." + ) + return self + # docs:method-name-end + + +class SequenceStep(CommonStep): + """Runs another sequence as a step.""" + + steptype: Literal["SequenceStep"] = described( + "Canonical registered step type.", example="SequenceStep" + ) + sequence: InternalSequenceReference = described( + "Internal sequence reference.", example={"type": "internal", "name": "Calibration"} + ) + + +class UserInteractionStep(CommonStep): + """Displays an operator interaction prompt.""" + + steptype: Literal["UserInteractionStep"] = described( + "Canonical registered step type.", example="UserInteractionStep" + ) + + +class WaitStep(CommonStep): + """Waits for a non-negative duration in seconds.""" + + steptype: Literal["WaitStep"] = described("Canonical registered step type.", example="WaitStep") + + # docs:wait-time-start + @model_validator(mode="after") + def has_wait_time(self) -> WaitStep: + if "wait_time" not in self.input_mapping: + raise PydanticCustomError("missing_required_input", "WaitStep requires input 'wait_time'.") + return self + # docs:wait-time-end + + +class UserLoadingStep(CommonStep): + """Prompts the operator to select a file.""" + + steptype: Literal["UserLoadingStep"] = described( + "Canonical registered step type.", example="UserLoadingStep" + ) + file_save_location: FileDestination | None = described( + "Local or global destination for the selected file.", + example={"type": "local", "variable": "selected_file"}, + default=None, + ) + + +class UserRunMethodStep(CommonStep): + """Optionally runs a Python method after an operator response.""" + + steptype: Literal["UserRunMethodStep"] = described( + "Canonical registered step type.", example="UserRunMethodStep" + ) + trigger_response: str | list[Any] | dict[str, Any] | None = described( + "Operator response that triggers execution.", example="run", default=None + ) + action_type: str | None = described("Optional Python action type.", example="method", default=None) + module: str | None = described("Optional Python module path.", example="tests.py", default=None) + method_name: str | None = described("Optional Python method name.", example="run", default=None) + + +class UserWriteStep(CommonStep): + """Writes an operator-provided value to a configured destination.""" + + steptype: Literal["UserWriteStep"] = described( + "Canonical registered step type.", example="UserWriteStep" + ) + + +class SerialNumberStep(CommonStep): + """Captures the device serial number.""" + + steptype: Literal["SerialNumberStep"] = described( + "Canonical registered step type.", example="SerialNumberStep" + ) + + +class SSHConnectStep(CommonStep): + """Opens the SSH client stored in recipe globals.""" + + steptype: Literal["SSHConnectStep"] = described( + "Canonical registered step type.", example="SSHConnectStep" + ) + + +class SSHCloseStep(CommonStep): + """Closes the SSH client stored in recipe globals.""" + + steptype: Literal["SSHCloseStep"] = described( + "Canonical registered step type.", example="SSHCloseStep" + ) + + +class SSHUploadStep(CommonStep): + """Uploads files through an SSH connection.""" + + steptype: Literal["SSHUploadStep"] = described( + "Canonical registered step type.", example="SSHUploadStep" + ) + files: list[UploadFile] = described( + "Local and remote file pairs to upload.", + example=[{"local": "bin/tool", "remote": "/tmp/tool"}], + ) + permissions: int | str | None = described( + "Optional remote permissions.", example="0755", default=None + ) + skip_if_sha256_match: bool = described( + "Skip files whose remote checksum matches.", example=False, default=False + ) + local_package: str | None = described( + "Optional package containing local resources.", example="my_package", default=None + ) + + +type Step = Annotated[ + PythonModuleStep + | SequenceStep + | UserInteractionStep + | WaitStep + | UserLoadingStep + | UserRunMethodStep + | UserWriteStep + | SerialNumberStep + | SSHConnectStep + | SSHCloseStep + | SSHUploadStep, + Field(discriminator="steptype"), +] + + +class RecipeHeader(RecipeModel): + """The first YAML document, identifying a recipe and its entry sequence.""" + + name: str = described("Human-readable recipe name.", example="Hardware acceptance") + version: str = described("Version of this recipe.", example="1.0") + recipe_version: Literal["2.0.0"] = described( + "Version of the recipe language contract.", example="2.0.0" + ) + description: str = described("Purpose of the recipe.", example="Acceptance tests.") + main_sequence: str = described("Sequence where execution begins.", example="Main") + globals: dict[str, Any] = described("Recipe-wide variables.", example={}) + continue_on_error: bool | None = described( + "Recipe-wide error policy.", example=False, default=None + ) + report: Literal["overwrite", "append"] = described( + "Report file mode.", example="overwrite", default="overwrite" + ) + report_name_include_serial: bool = described( + "Include the serial number in the report name.", example=False, default=False + ) + test_package: str | None = described( + "Package containing recipe test modules.", example="acceptance", default=None + ) + + +class Sequence(RecipeModel): + """One named executable sequence document.""" + + sequence_name: str = described("Unique sequence name.", example="Main") + description: str = described("Purpose of the sequence.", example="Main sequence.") + parameters: dict[str, Any] = described("Reserved sequence input metadata.", example={}) + outputs: dict[str, Any] = described("Reserved sequence output metadata.", example={}) + locals: dict[str, Any] = described("Variables local to the sequence.", example={}) + setup_steps: list[Step] = described("Steps run before the main steps.", example=[]) + steps: list[Step] = described("Ordered main steps.", example=[]) + teardown_steps: list[Step] = described("Steps run during teardown.", example=[]) + + +class Recipe(RecipeModel): + """Aggregate typed recipe used by tooling and JSON Schema consumers.""" + + header: RecipeHeader = described("Recipe header document.") + sequences: list[Sequence] = described("Sequence documents.", min_length=1) + + +def _union_models(annotation: Any) -> tuple[type[RecipeModel], ...]: + """Expose union members for generators without a second registry.""" + annotation = getattr(annotation, "__value__", annotation) + annotated_union = get_args(annotation)[0] + return get_args(annotated_union) + + +STEP_MODELS = _union_models(Step) +INPUT_MODELS = _union_models(InputMapping) +OUTPUT_MODELS = _union_models(OutputMapping) diff --git a/src/pypts/recipe_parser.py b/src/pypts/recipe_parser.py index 3d25be4..680b6b5 100644 --- a/src/pypts/recipe_parser.py +++ b/src/pypts/recipe_parser.py @@ -1,212 +1,60 @@ # SPDX-FileCopyrightText: 2026 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -"""Safe, source-aware parsing for the pypts recipe language. - -The parser is intentionally isolated from recipe execution and GUI code. It -turns YAML into immutable definitions after validation by -``pypts.recipe_language``. -""" +"""Safe YAML adapter and semantic validation for recipe language 2.0.0.""" from __future__ import annotations -from collections.abc import Iterator, Mapping -from dataclasses import dataclass, field, replace +from collections.abc import Mapping +from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeAlias +from typing import Any import yaml +from pydantic import ValidationError from pypts.recipe_language import ( - Diagnostic, - SourcePosition, - SourceSpan, - STEP_SPECS_BY_NAME, - canonical_step_type, - validate_recipe_documents, + DirectInput, + EqualsOutput, + PassFailOutput, + PassthroughOutput, + RangeOutput, + Recipe, + RecipeHeader, + Sequence, + SequenceStep, ) - -RecipePath: TypeAlias = tuple[str | int, ...] - - -def _freeze(value: Any) -> Any: - if isinstance(value, Mapping): - return FrozenMap((str(key), _freeze(item)) for key, item in value.items()) - if isinstance(value, list | tuple): - return tuple(_freeze(item) for item in value) - if isinstance(value, set | frozenset): - return frozenset(_freeze(item) for item in value) - return value - - -def _thaw(value: Any) -> Any: - if isinstance(value, FrozenMap): - return {key: _thaw(item) for key, item in value.items()} - if isinstance(value, tuple): - return [_thaw(item) for item in value] - if isinstance(value, frozenset): - return set(_thaw(item) for item in value) - return value - - -@dataclass(frozen=True, eq=False) -class FrozenMap(Mapping[str, Any]): - """Small insertion-ordered immutable mapping used by parsed models.""" - - entries: tuple[tuple[str, Any], ...] = () - - def __init__(self, entries=()): - object.__setattr__(self, "entries", tuple(entries)) - - @classmethod - def from_mapping(cls, value: Mapping[str, Any] | None) -> "FrozenMap": - return cls((str(key), _freeze(item)) for key, item in (value or {}).items()) - - def __getitem__(self, key: str) -> Any: - for candidate, value in self.entries: - if candidate == key: - return value - raise KeyError(key) - - def __iter__(self) -> Iterator[str]: - return (key for key, _ in self.entries) - - def __len__(self) -> int: - return len(self.entries) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Mapping) or len(self) != len(other): - return False - return all(key in other and value == other[key] for key, value in self.items()) - - def __hash__(self) -> int: - return hash(frozenset(self.entries)) - - -@dataclass(frozen=True) -class DirectInput: - value: Any - indexed: bool = False - span: SourceSpan | None = field(default=None, compare=False) +type RecipePath = tuple[str | int, ...] @dataclass(frozen=True) -class LocalInput: - local_name: str - span: SourceSpan | None = field(default=None, compare=False) +class SourcePosition: + """A one-based source position with a zero-based character offset.""" - -@dataclass(frozen=True) -class GlobalInput: - global_name: str - span: SourceSpan | None = field(default=None, compare=False) + line: int + column: int + offset: int @dataclass(frozen=True) -class MethodInput: - value: Any - span: SourceSpan | None = field(default=None, compare=False) +class SourceSpan: + """Half-open source range.""" - -InputDefinition: TypeAlias = DirectInput | LocalInput | GlobalInput | MethodInput + start: SourcePosition + end: SourcePosition @dataclass(frozen=True) -class PassFailOutput: - span: SourceSpan | None = field(default=None, compare=False) - +class Diagnostic: + """Source-aware recipe finding, compatible with the PyPTS envelope.""" -@dataclass(frozen=True) -class EqualsOutput: - value: Any - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class RangeOutput: - minimum: Any - maximum: Any - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class PassthroughOutput: - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class LocalOutput: - local_name: str - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class GlobalOutput: - global_name: str - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class ImageOutput: - span: SourceSpan | None = field(default=None, compare=False) - - -OutputDefinition: TypeAlias = ( - PassFailOutput | EqualsOutput | RangeOutput | PassthroughOutput | - LocalOutput | GlobalOutput | ImageOutput -) - - -@dataclass(frozen=True) -class StepDefinition: - steptype: str - step_name: str - description: str - id: str | None - skip: bool - critical: bool - continue_on_error: bool - input_mapping: FrozenMap - output_mapping: FrozenMap - configuration: FrozenMap - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class SequenceDefinition: - sequence_name: str - description: str - parameters: FrozenMap - outputs: FrozenMap - locals: FrozenMap - setup_steps: tuple[StepDefinition, ...] - steps: tuple[StepDefinition, ...] - teardown_steps: tuple[StepDefinition, ...] - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class RecipeHeader: - name: str - version: str - recipe_version: str - description: str - main_sequence: str - globals: FrozenMap - continue_on_error: bool | None - report: str - report_name_include_serial: bool - test_package: str | None - span: SourceSpan | None = field(default=None, compare=False) - - -@dataclass(frozen=True) -class RecipeDefinition: - header: RecipeHeader - sequences: tuple[SequenceDefinition, ...] - source_name: str = field(default="", compare=False) - span: SourceSpan | None = field(default=None, compare=False) + code: str + message: str + path: RecipePath = () + severity: str = "error" + source_name: str | None = None + span: SourceSpan | None = None class RecipeParseError(ValueError): @@ -220,7 +68,7 @@ def __init__(self, diagnostics: tuple[Diagnostic, ...]): @dataclass(frozen=True) class ParseResult: - recipe: RecipeDefinition | None + recipe: Recipe | None diagnostics: tuple[Diagnostic, ...] = () @property @@ -235,9 +83,10 @@ def warnings(self) -> tuple[Diagnostic, ...]: def is_valid(self) -> bool: return self.recipe is not None and not self.errors - def require_recipe(self) -> RecipeDefinition: - if self.recipe is None or self.errors: + def require_recipe(self) -> Recipe: + if not self.is_valid: raise RecipeParseError(self.diagnostics) + assert self.recipe is not None return self.recipe @@ -256,6 +105,25 @@ def _mark_span(mark: yaml.error.Mark | None) -> SourceSpan | None: return SourceSpan(position, position) +def _nearest_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: + candidate = path + while candidate: + if candidate in spans: + return spans[candidate] + candidate = candidate[:-1] + return spans.get(()) + + +def _diagnostic( + code: str, + message: str, + path: RecipePath, + source_name: str, + spans: Mapping[RecipePath, SourceSpan], +) -> Diagnostic: + return Diagnostic(code, message, path, source_name=source_name, span=_nearest_span(path, spans)) + + def _index_nodes( node: yaml.Node, path: RecipePath, @@ -265,226 +133,320 @@ def _index_nodes( active: set[int], ) -> None: spans[path] = _span(node) - node_id = id(node) - if node_id in active: + identity = id(node) + if identity in active: diagnostics.append(Diagnostic( - "recursive-alias", "Recursive YAML aliases are not supported.", path, - source_name=source_name, span=_span(node), + "recursive-alias", + "Recursive YAML aliases are not supported.", + path, + source_name=source_name, + span=_span(node), )) return - active.add(node_id) + active.add(identity) try: if isinstance(node, yaml.MappingNode): seen: set[tuple[str, str]] = set() for key_node, value_node in node.value: - if isinstance(key_node, yaml.ScalarNode): - identity = (key_node.tag, key_node.value) - key: str | int = key_node.value - else: - identity = (key_node.tag, repr(key_node.value)) - key = repr(key_node.value) - child_path = path + (key,) - if identity in seen: + key_identity = (key_node.tag, repr(key_node.value)) + key: str | int = key_node.value if isinstance(key_node, yaml.ScalarNode) else repr(key_node.value) + child = path + (key,) + if key_identity in seen: diagnostics.append(Diagnostic( - "duplicate-key", f"Duplicate YAML key '{key}'.", child_path, - source_name=source_name, span=_span(key_node), + "duplicate-key", + f"Duplicate YAML key '{key}'.", + child, + source_name=source_name, + span=_span(key_node), )) - seen.add(identity) - spans[child_path] = _span(value_node) - _index_nodes(value_node, child_path, spans, diagnostics, source_name, active) + seen.add(key_identity) + _index_nodes(value_node, child, spans, diagnostics, source_name, active) elif isinstance(node, yaml.SequenceNode): - for index, child in enumerate(node.value): - _index_nodes(child, path + (index,), spans, diagnostics, source_name, active) + for index, child_node in enumerate(node.value): + _index_nodes( + child_node, path + (index,), spans, diagnostics, source_name, active + ) finally: - active.remove(node_id) + active.remove(identity) -def _nearest_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: - candidate = path - while candidate: - if candidate in spans: - return spans[candidate] - candidate = candidate[:-1] - return spans.get(()) +_MODEL_TAGS = { + "direct", "local", "global", "method", "passfail", "equals", "range", + "passthrough", "image", "PythonModuleStep", "SequenceStep", + "UserInteractionStep", "WaitStep", "UserLoadingStep", "UserRunMethodStep", + "UserWriteStep", "SerialNumberStep", "SSHConnectStep", "SSHCloseStep", + "SSHUploadStep", +} +_CANONICAL_STEPS = {tag for tag in _MODEL_TAGS if tag.endswith("Step")} -def _source_diagnostic(code: str, message: str, source_name: str, mark=None) -> Diagnostic: - return Diagnostic(code, message, source_name=source_name, span=_mark_span(mark)) +def _clean_location(location: tuple[Any, ...]) -> RecipePath: + cleaned: list[str | int] = [] + for index, item in enumerate(location): + is_step_tag = ( + item in _CANONICAL_STEPS + and index >= 2 + and location[index - 2] in {"setup_steps", "steps", "teardown_steps"} + and isinstance(location[index - 1], int) + ) + is_mapping_tag = ( + item in _MODEL_TAGS + and index >= 2 + and location[index - 2] in {"input_mapping", "output_mapping"} + ) + if not is_step_tag and not is_mapping_tag: + cleaned.append(item) + return tuple(cleaned) -def _enrich_diagnostics( - diagnostics: tuple[Diagnostic, ...], +def _pydantic_diagnostic( + error: dict[str, Any], + prefix: RecipePath, source_name: str, spans: Mapping[RecipePath, SourceSpan], - sequence_documents: Mapping[str, int], -) -> list[Diagnostic]: - enriched: list[Diagnostic] = [] - for item in diagnostics: - path = item.path - if path and isinstance(path[0], str) and path[0] in sequence_documents: - path = (sequence_documents[path[0]],) + path[1:] - enriched.append(replace( - item, - source_name=item.source_name or source_name, - span=item.span or _nearest_span(path, spans), - )) - return enriched - - -def _normalization_warnings( - documents: list[Any], +) -> Diagnostic: + location = prefix + _clean_location(tuple(error.get("loc", ()))) + kind = error["type"] + context = error.get("ctx") or {} + input_value = error.get("input") + code = "invalid-field" + message = error["msg"] + + if kind == "missing": + code = "missing-field" + elif kind == "extra_forbidden": + field = location[-1] if location else "field" + if field == "serial_number": + code = "removed-sequence-field" + message = "Sequence field 'serial_number' was removed in recipe language 2.0.0." + else: + code = "unknown-field" + message = f"Unknown field '{field}'." + elif kind == "union_tag_not_found": + discriminator = str(context.get("discriminator", "")) + if "steptype" in discriminator: + code = "missing-step-type" + location += ("steptype",) + message = "Step requires canonical 'steptype'." + elif "output_mapping" in location: + code = "missing-output-type" + location += ("type",) + message = "Output mapping requires an explicit 'type'." + else: + code = "missing-input-type" + location += ("type",) + message = "Mapping requires an explicit 'type' in recipe language 2.0.0." + elif kind == "union_tag_invalid": + discriminator = str(context.get("discriminator", "")) + tag = context.get("tag") + if "steptype" in discriminator: + location += ("steptype",) + canonical = next( + (candidate for candidate in _CANONICAL_STEPS if candidate.casefold() == str(tag).casefold()), + None, + ) + if canonical: + code = "noncanonical-step-type" + message = f"Use canonical step type '{canonical}' instead of '{tag}'." + else: + code = "unknown-step-type" + message = f"Unknown step type '{tag}'." + else: + location += ("type",) + if "output_mapping" in location: + code = "unknown-output-type" + message = f"Unknown output mapping type '{tag}'." + else: + code = "unknown-input-type" + message = f"Unknown input mapping type '{tag}'." + elif kind == "literal_error" and location[-1:] == ("recipe_version",): + code = "unsupported-recipe-version" + message = f"Recipe language version {input_value!r} is unsupported; expected '2.0.0'." + elif kind == "literal_error": + code = "invalid-field-value" + elif kind in {"bool_type", "string_type", "int_type", "list_type", "dict_type", "model_type"}: + code = "invalid-field-type" + elif kind == "invalid_indexed_input": + code = "invalid-indexed-input" + elif kind == "missing_method_name": + code = "missing-method-name" + location += ("method_name",) + elif kind == "missing_required_input": + code = "missing-required-input" + location += ("input_mapping", "wait_time") + elif kind == "too_short" and prefix == () and location[-1:] == ("sequences",): + code = "missing-sequence" + + return _diagnostic(code, message, location, source_name, spans) + + +def _validation_diagnostics( + error: ValidationError, + prefix: RecipePath, source_name: str, spans: Mapping[RecipePath, SourceSpan], ) -> list[Diagnostic]: - warnings: list[Diagnostic] = [] - for doc_index, document in enumerate(documents[1:], start=1): - if not isinstance(document, Mapping): - continue - for section in ("setup_steps", "steps", "teardown_steps"): - values = document.get(section, []) - if not isinstance(values, list): - continue - for step_index, step in enumerate(values): - if not isinstance(step, Mapping): - continue - step_path = (doc_index, section, step_index) - raw_type = step.get("steptype") - canonical = canonical_step_type(raw_type) - if canonical and raw_type != canonical: - path = step_path + ("steptype",) - warnings.append(Diagnostic( - "noncanonical-step-type", - f"Use canonical step type '{canonical}' instead of '{raw_type}'.", - path, "warning", source_name, _nearest_span(path, spans), - )) - mapping = step.get("input_mapping", {}) - if isinstance(mapping, Mapping): - for input_name, config in mapping.items(): - if isinstance(config, Mapping) and "type" not in config: - path = step_path + ("input_mapping", input_name) - warnings.append(Diagnostic( - "implicit-direct-input", - f"Input '{input_name}' omits type; it is normalized to 'direct'.", - path, "warning", source_name, _nearest_span(path, spans), - )) - return warnings - - -def _mapping_span(path: RecipePath, spans: Mapping[RecipePath, SourceSpan]) -> SourceSpan | None: - return _nearest_span(path, spans) - - -def _build_input(config: Mapping[str, Any], path: RecipePath, spans) -> InputDefinition: - kind = config.get("type", "direct") - item_span = _mapping_span(path, spans) - if kind == "direct": - return DirectInput(_freeze(config["value"]), bool(config.get("indexed", False)), item_span) - if kind == "local": - return LocalInput(config["local_name"], item_span) - if kind == "global": - return GlobalInput(config["global_name"], item_span) - return MethodInput(_freeze(config["value"]), item_span) - - -def _build_output(config: Mapping[str, Any], path: RecipePath, spans) -> OutputDefinition: - kind = config["type"] - item_span = _mapping_span(path, spans) - if kind == "passfail": - return PassFailOutput(item_span) - if kind == "equals": - return EqualsOutput(_freeze(config["value"]), item_span) - if kind == "range": - return RangeOutput(_freeze(config["min"]), _freeze(config["max"]), item_span) - if kind == "passthrough": - return PassthroughOutput(item_span) - if kind == "local": - return LocalOutput(config["local_name"], item_span) - if kind == "global": - return GlobalOutput(config["global_name"], item_span) - return ImageOutput(item_span) - - -_COMMON_STEP_KEYS = { - "steptype", "step_name", "description", "id", "skip", "critical", - "continue_on_error", "input_mapping", "output_mapping", -} - - -def _build_step(step: Mapping[str, Any], path: RecipePath, spans) -> StepDefinition: - canonical = canonical_step_type(step["steptype"]) - assert canonical is not None - inputs = FrozenMap( - (str(name), _build_input(config, path + ("input_mapping", name), spans)) - for name, config in step.get("input_mapping", {}).items() - ) - outputs = FrozenMap( - (str(name), _build_output(config, path + ("output_mapping", name), spans)) - for name, config in step.get("output_mapping", {}).items() - ) - configuration = FrozenMap( - (name, _freeze(value)) for name, value in step.items() - if name not in _COMMON_STEP_KEYS - ) - return StepDefinition( - canonical, step["step_name"], step["description"], step.get("id"), - bool(step.get("skip", False)), bool(step.get("critical", False)), - bool(step.get("continue_on_error", False)), inputs, outputs, - configuration, _mapping_span(path, spans), - ) + return [ + _pydantic_diagnostic(item, prefix, source_name, spans) + for item in error.errors(include_url=False) + ] -def _build_sequence(sequence: Mapping[str, Any], doc_index: int, spans) -> SequenceDefinition: - def build_section(name: str) -> tuple[StepDefinition, ...]: - return tuple( - _build_step(step, (doc_index, name, index), spans) - for index, step in enumerate(sequence.get(name, [])) - ) - return SequenceDefinition( - sequence["sequence_name"], sequence["description"], - FrozenMap.from_mapping(sequence["parameters"]), - FrozenMap.from_mapping(sequence["outputs"]), - FrozenMap.from_mapping(sequence["locals"]), - build_section("setup_steps"), build_section("steps"), - build_section("teardown_steps"), _mapping_span((doc_index,), spans), - ) +def _all_steps(sequence: Sequence): + for section_name in ("setup_steps", "steps", "teardown_steps"): + for index, step in enumerate(getattr(sequence, section_name)): + yield section_name, index, step -def _build_recipe(documents: list[Mapping[str, Any]], source_name: str, spans) -> RecipeDefinition: - raw = documents[0] - header = RecipeHeader( - raw["name"], raw["version"], raw["recipe_version"], raw["description"], - raw["main_sequence"], FrozenMap.from_mapping(raw["globals"]), - raw.get("continue_on_error"), raw.get("report", "overwrite"), - bool(raw.get("report_name_include_serial", False)), raw.get("test_package"), - _mapping_span((0,), spans), - ) - sequences = tuple( - _build_sequence(sequence, index, spans) - for index, sequence in enumerate(documents[1:], start=1) - ) - recipe_span = None - if documents: - first = spans.get((0,)) - last = spans.get((len(documents) - 1,)) - if first and last: - recipe_span = SourceSpan(first.start, last.end) - return RecipeDefinition(header, sequences, source_name, recipe_span) +def _semantic_diagnostics( + header: RecipeHeader | None, + sequences: list[tuple[int, Sequence]], + source_name: str, + spans: Mapping[RecipePath, SourceSpan], + *, + complete_sequences: bool = True, +) -> list[Diagnostic]: + """Rules that cannot be expressed by one structural Pydantic model.""" + diagnostics: list[Diagnostic] = [] + # docs:sequence-semantics-start + by_name: dict[str, tuple[int, Sequence]] = {} + for document_index, sequence in sequences: + path = (document_index, "sequence_name") + if sequence.sequence_name in by_name: + diagnostics.append(_diagnostic( + "duplicate-sequence", + f"Duplicate sequence '{sequence.sequence_name}'.", + path, + source_name, + spans, + )) + else: + by_name[sequence.sequence_name] = (document_index, sequence) + + if complete_sequences and header is not None and header.main_sequence not in by_name: + diagnostics.append(_diagnostic( + "unknown-main-sequence", + f"Main sequence '{header.main_sequence}' does not exist.", + (0, "main_sequence"), + source_name, + spans, + )) + # docs:sequence-semantics-end + + verdict_types = (PassFailOutput, EqualsOutput, RangeOutput, PassthroughOutput) + for document_index, sequence in sequences: + flattened = list(_all_steps(sequence)) + for section, index, step in flattened: + step_path = (document_index, section, index) + # docs:nested-reference-start + if isinstance(step, SequenceStep) and step.sequence.name not in by_name: + diagnostics.append(_diagnostic( + "unknown-sequence-reference", + f"Sequence '{sequence.sequence_name}' references unknown sequence " + f"'{step.sequence.name}'.", + step_path + ("sequence", "name"), + source_name, + spans, + )) + # docs:nested-reference-end + + # docs:mapping-semantics-start + indexed_lengths = [ + len(value.value) + for value in step.input_mapping.values() + if isinstance(value, DirectInput) and value.indexed + ] + if len(set(indexed_lengths)) > 1: + diagnostics.append(_diagnostic( + "unequal-indexed-inputs", + "Indexed input lists must have equal lengths.", + step_path + ("input_mapping",), + source_name, + spans, + )) + + verdicts = [ + value for value in step.output_mapping.values() if isinstance(value, verdict_types) + ] + if any(isinstance(value, PassthroughOutput) for value in verdicts) and len(verdicts) != 1: + diagnostics.append(_diagnostic( + "mixed-passthrough", + "'passthrough' must be the sole verdict mapping.", + step_path + ("output_mapping",), + source_name, + spans, + )) + # docs:mapping-semantics-end + + # docs:ssh-semantics-start + ssh_steps = [item for item in flattened if item[2].steptype.startswith("SSH")] + if ssh_steps and header is not None: + for required in ("ssh_client", "host", "user", "port"): + if required not in header.globals: + diagnostics.append(_diagnostic( + "missing-ssh-global", + f"SSH step requires global '{required}'.", + (0, "globals", required), + source_name, + spans, + )) + if "password" not in header.globals and "private_key" not in header.globals: + diagnostics.append(_diagnostic( + "missing-ssh-credential", + "SSH steps require password or private_key global.", + (0, "globals"), + source_name, + spans, + )) + + connected = False + unclosed_connect: tuple[str, int] | None = None + for section, index, step in flattened: + if step.steptype == "SSHConnectStep": + connected = True + unclosed_connect = (section, index) + elif step.steptype == "SSHUploadStep" and not connected: + diagnostics.append(_diagnostic( + "missing-ssh-connect", + f"Sequence '{sequence.sequence_name}' uploads before an SSH connection.", + (document_index, section, index), + source_name, + spans, + )) + elif step.steptype == "SSHCloseStep": + connected = False + unclosed_connect = None + if unclosed_connect is not None: + diagnostics.append(_diagnostic( + "missing-ssh-close", + f"Sequence '{sequence.sequence_name}' opens SSH without a later close.", + (document_index, "teardown_steps"), + source_name, + spans, + )) + # docs:ssh-semantics-end + return diagnostics def parse_recipe_text(text: str, source_name: str = "") -> ParseResult: - """Parse recipe YAML text without importing or invoking the runtime.""" + """Parse candidate recipe-language 2 YAML without runtime or GUI imports.""" if not isinstance(text, str): - diagnostic = Diagnostic("invalid-source", "Recipe source must be text.", source_name=source_name) - return ParseResult(None, (diagnostic,)) + return ParseResult(None, (Diagnostic( + "invalid-source", "Recipe source must be text.", source_name=source_name + ),)) if not text.strip(): - diagnostic = Diagnostic("empty-recipe", "A recipe requires a header and at least one sequence.", source_name=source_name) - return ParseResult(None, (diagnostic,)) + return ParseResult(None, (Diagnostic( + "empty-recipe", "A recipe requires a header and at least one sequence.", + source_name=source_name, + ),)) try: nodes = list(yaml.compose_all(text, Loader=yaml.SafeLoader)) except yaml.YAMLError as error: mark = getattr(error, "problem_mark", None) - return ParseResult(None, (_source_diagnostic("yaml-syntax-error", str(error), source_name, mark),)) + return ParseResult(None, (Diagnostic( + "yaml-syntax-error", str(error), source_name=source_name, span=_mark_span(mark) + ),)) spans: dict[RecipePath, SourceSpan] = {} diagnostics: list[Diagnostic] = [] @@ -497,121 +459,87 @@ def parse_recipe_text(text: str, source_name: str = "") -> ParseResult: except yaml.YAMLError as error: mark = getattr(error, "problem_mark", None) code = "unsafe-yaml" if isinstance(error, yaml.constructor.ConstructorError) else "yaml-construction-error" - diagnostics.append(_source_diagnostic(code, str(error), source_name, mark)) + diagnostics.append(Diagnostic(code, str(error), source_name=source_name, span=_mark_span(mark))) + return ParseResult(None, tuple(diagnostics)) + + if not documents or all(document is None for document in documents): + diagnostics.append(Diagnostic( + "empty-recipe", "A recipe requires a header and at least one sequence.", + source_name=source_name, + )) return ParseResult(None, tuple(diagnostics)) - sequence_documents = { - document["sequence_name"]: index - for index, document in enumerate(documents) - if isinstance(document, Mapping) and isinstance(document.get("sequence_name"), str) - } - contract = validate_recipe_documents(documents) - diagnostics.extend(_enrich_diagnostics(contract.diagnostics, source_name, spans, sequence_documents)) - diagnostics.extend(_normalization_warnings(documents, source_name, spans)) - if any(item.severity == "error" for item in diagnostics): + header: RecipeHeader | None = None + semantic_header: RecipeHeader | None = None + raw_header = documents[0] + try: + header = RecipeHeader.model_validate(raw_header) + semantic_header = header + except ValidationError as error: + diagnostics.extend(_validation_diagnostics(error, (0,), source_name, spans)) + if isinstance(raw_header, dict) and raw_header.get("recipe_version") != "2.0.0": + candidate = dict(raw_header) + candidate["recipe_version"] = "2.0.0" + try: + semantic_header = RecipeHeader.model_validate(candidate) + except ValidationError: + pass + + sequences: list[tuple[int, Sequence]] = [] + complete_sequences = True + for index, document in enumerate(documents[1:], start=1): + try: + sequences.append((index, Sequence.model_validate(document))) + except ValidationError as error: + complete_sequences = False + diagnostics.extend(_validation_diagnostics(error, (index,), source_name, spans)) + + if len(documents) == 1: + diagnostics.append(_diagnostic( + "missing-sequence", "A recipe requires at least one sequence.", (0,), source_name, spans + )) + + diagnostics.extend(_semantic_diagnostics( + semantic_header, + sequences, + source_name, + spans, + complete_sequences=complete_sequences, + )) + if diagnostics: return ParseResult(None, tuple(diagnostics)) - return ParseResult(_build_recipe(documents, source_name, spans), tuple(diagnostics)) + assert header is not None + recipe = Recipe(header=header, sequences=[sequence for _, sequence in sequences]) + return ParseResult(recipe) def parse_recipe_file(path: str | Path, encoding: str = "utf-8") -> ParseResult: - """Read and parse a recipe file, reporting read failures as diagnostics.""" + """Read and parse a candidate recipe file.""" source_path = Path(path) - source_name = str(source_path) try: text = source_path.read_text(encoding=encoding) except (OSError, UnicodeError) as error: return ParseResult(None, (Diagnostic( - "file-read-error", f"Could not read recipe: {error}", - source_name=source_name, + "file-read-error", f"Could not read recipe: {error}", source_name=str(source_path) ),)) - return parse_recipe_text(text, source_name) - - -def _input_to_mapping(value: InputDefinition) -> dict[str, Any]: - if isinstance(value, DirectInput): - result = {"type": "direct", "value": _thaw(value.value)} - if value.indexed: - result["indexed"] = True - return result - if isinstance(value, LocalInput): - return {"type": "local", "local_name": value.local_name} - if isinstance(value, GlobalInput): - return {"type": "global", "global_name": value.global_name} - return {"type": "method", "value": _thaw(value.value)} - - -def _output_to_mapping(value: OutputDefinition) -> dict[str, Any]: - if isinstance(value, PassFailOutput): - return {"type": "passfail"} - if isinstance(value, EqualsOutput): - return {"type": "equals", "value": _thaw(value.value)} - if isinstance(value, RangeOutput): - return {"type": "range", "min": _thaw(value.minimum), "max": _thaw(value.maximum)} - if isinstance(value, PassthroughOutput): - return {"type": "passthrough"} - if isinstance(value, LocalOutput): - return {"type": "local", "local_name": value.local_name} - if isinstance(value, GlobalOutput): - return {"type": "global", "global_name": value.global_name} - return {"type": "image"} - - -def _step_to_mapping(step: StepDefinition) -> dict[str, Any]: - result: dict[str, Any] = {"steptype": step.steptype, "step_name": step.step_name} - if step.id is not None: - result["id"] = step.id - result["description"] = step.description - result["skip"] = step.skip - result["critical"] = step.critical - result["continue_on_error"] = step.continue_on_error - spec = STEP_SPECS_BY_NAME[step.steptype.casefold()] - for field_spec in spec.fields: - if field_spec.name in _COMMON_STEP_KEYS or field_spec.name not in step.configuration: - continue - result[field_spec.name] = _thaw(step.configuration[field_spec.name]) - result["input_mapping"] = { - name: _input_to_mapping(value) for name, value in step.input_mapping.items() - } - result["output_mapping"] = { - name: _output_to_mapping(value) for name, value in step.output_mapping.items() - } - return result - - -def _sequence_to_mapping(sequence: SequenceDefinition) -> dict[str, Any]: - return { - "sequence_name": sequence.sequence_name, - "description": sequence.description, - "parameters": _thaw(sequence.parameters), - "outputs": _thaw(sequence.outputs), - "locals": _thaw(sequence.locals), - "setup_steps": [_step_to_mapping(step) for step in sequence.setup_steps], - "steps": [_step_to_mapping(step) for step in sequence.steps], - "teardown_steps": [_step_to_mapping(step) for step in sequence.teardown_steps], - } - - -def dump_recipe(recipe: RecipeDefinition) -> str: - """Serialize a typed recipe to stable canonical multi-document YAML.""" - if not isinstance(recipe, RecipeDefinition): - raise TypeError("dump_recipe expects a RecipeDefinition") - header = recipe.header - header_document: dict[str, Any] = { - "name": header.name, - "version": header.version, - "recipe_version": header.recipe_version, - "description": header.description, - "main_sequence": header.main_sequence, - } - if header.test_package is not None: - header_document["test_package"] = header.test_package - if header.continue_on_error is not None: - header_document["continue_on_error"] = header.continue_on_error - header_document["report"] = header.report - header_document["report_name_include_serial"] = header.report_name_include_serial - header_document["globals"] = _thaw(header.globals) - documents = [header_document] + [_sequence_to_mapping(sequence) for sequence in recipe.sequences] + return parse_recipe_text(text, str(source_path)) + + +def dump_recipe(recipe: Recipe) -> str: + """Serialize a typed recipe as canonical multi-document YAML.""" + if not isinstance(recipe, Recipe): + raise TypeError("dump_recipe expects a Recipe") + documents = [ + recipe.header.model_dump(mode="python", by_alias=True, exclude_none=True), + *[ + sequence.model_dump(mode="python", by_alias=True, exclude_none=True) + for sequence in recipe.sequences + ], + ] return yaml.safe_dump_all( - documents, explicit_start=True, sort_keys=False, - default_flow_style=False, allow_unicode=True, + documents, + explicit_start=True, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, ) diff --git a/src/pypts/recipe_reference.py b/src/pypts/recipe_reference.py index c966fa1..823b0ee 100644 --- a/src/pypts/recipe_reference.py +++ b/src/pypts/recipe_reference.py @@ -1,335 +1,216 @@ # SPDX-FileCopyrightText: 2026 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -"""Generate the standalone reference for the pypts recipe language.""" +"""Render the Sphinx recipe reference from generated JSON Schema only.""" from __future__ import annotations -import argparse -import sys -from collections.abc import Mapping, Sequence +import json from pathlib import Path from typing import Any -import yaml -from pypts.recipe_language import ( - CANONICAL_RECIPE_VERSION, - COMMON_STEP_FIELDS, - CONSTRAINT_SPECS, - HEADER_SPEC, - INPUT_MAPPING_SPECS, - OUTPUT_MAPPING_SPECS, - SEQUENCE_SPEC, - STEP_SPECS, - FieldSpec, - MappingSpec, - StepSpec, -) -from pypts.recipe_parser import dump_recipe, parse_recipe_text - -DEFAULT_REFERENCE_PATH = Path("docs/generated/recipe_language_reference.rst") - - -def _header() -> dict[str, Any]: - return { - "name": "Recipe language reference fixture", - "version": "1.0", - "recipe_version": CANONICAL_RECIPE_VERSION, - "description": "Executable generated-reference fixture.", - "main_sequence": "Main", - "globals": { - "ssh_client": None, - "host": "target", - "user": "root", - "port": 22, - "password": "secret", - }, - } - - -def _sequence(name: str, steps: list[Mapping[str, Any]] | None = None) -> dict[str, Any]: +def load_schema(path: str | Path) -> dict[str, Any]: + """Load and minimally verify an aggregate recipe JSON Schema.""" + source = Path(path) + schema = json.loads(source.read_text(encoding="utf-8")) + if not isinstance(schema, dict) or not isinstance(schema.get("$defs"), dict): + raise TypeError(f"Recipe schema has no $defs object: {source}") + return schema + + +def _reference_name(reference: str) -> str: + prefix = "#/$defs/" + if not reference.startswith(prefix): + raise ValueError(f"Unsupported external JSON Schema reference: {reference}") + return reference.removeprefix(prefix) + + +def _resolve(value: dict[str, Any], definitions: dict[str, Any]) -> dict[str, Any]: + if "$ref" not in value: + return value + return definitions[_reference_name(value["$ref"])] + + +def _type_name(value: dict[str, Any]) -> str: + if "$ref" in value: + return _reference_name(value["$ref"]) + if "const" in value: + return repr(value["const"]) + if "enum" in value: + return " | ".join(repr(item) for item in value["enum"]) + if "anyOf" in value: + return " | ".join(_type_name(item) for item in value["anyOf"]) + kind = value.get("type") + if kind == "array": + return f"list[{_type_name(value.get('items', {}))}]" + if kind == "object": + additional = value.get("additionalProperties") + if isinstance(additional, dict): + return f"dict[str, {_type_name(additional)}]" + return "object" return { - "sequence_name": name, - "description": f"{name} sequence.", - "parameters": {}, - "outputs": {}, - "locals": {"local_value": 1}, - "setup_steps": [], - "steps": list(steps or []), - "teardown_steps": [], - } - - -def _documents_with_step(step: Mapping[str, Any]) -> list[dict[str, Any]]: - authored = dict(step) - main = _sequence("Main", [authored]) - documents = [_header(), main] - if authored.get("steptype") == "SequenceStep": - target = authored.get("sequence", {}).get("name") - if isinstance(target, str) and target != "Main": - documents.append(_sequence(target)) - if authored.get("steptype") == "SSHUploadStep": - main["setup_steps"] = [{ - "steptype": "SSHConnectStep", - "step_name": "Connect", - "description": "Open the fixture connection.", - }] - main["teardown_steps"] = [{ - "steptype": "SSHCloseStep", - "step_name": "Close", - "description": "Close the fixture connection.", - }] - return documents - - -def _fixture_text(step: Mapping[str, Any]) -> str: - return yaml.safe_dump_all( - _documents_with_step(step), - explicit_start=True, - sort_keys=False, - allow_unicode=True, - ) - - -def _canonical_step_example(spec: StepSpec) -> Mapping[str, Any]: - result = parse_recipe_text(_fixture_text(spec.example), f"reference:{spec.name}") - if result.diagnostics: - details = "; ".join(f"{item.code}: {item.message}" for item in result.diagnostics) - raise ValueError(f"Invalid canonical example for {spec.name}: {details}") - documents = list(yaml.safe_load_all(dump_recipe(result.require_recipe()))) - main = next(document for document in documents[1:] if document["sequence_name"] == "Main") - return main["steps"][0] - - -def _validate_mapping_example(spec: MappingSpec, *, output: bool) -> None: - step: dict[str, Any] = { - "steptype": "PythonModuleStep", - "step_name": f"{spec.name} mapping", - "description": "Executable mapping fixture.", - "action_type": "method", - "module": "tests.py", - "method_name": "run", - "input_mapping": {}, - "output_mapping": {}, - } - mapping_name = "output_mapping" if output else "input_mapping" - step[mapping_name] = {"example": dict(spec.example)} - result = parse_recipe_text(_fixture_text(step), f"reference:{mapping_name}:{spec.name}") - if result.diagnostics: - details = "; ".join(f"{item.code}: {item.message}" for item in result.diagnostics) - raise ValueError(f"Invalid {mapping_name} example for {spec.name}: {details}") - - -def _type_name(field: FieldSpec) -> str: - if field.value_type is None: - return "any" - values = field.value_type if isinstance(field.value_type, tuple) else (field.value_type,) - return " or ".join(value.__name__ for value in values) + "boolean": "bool", + "integer": "int", + "number": "number", + "null": "None", + "string": "str", + }.get(kind, "any") def _literal(value: Any) -> str: - if value == {}: - return "{}" - if value is None: - return "null" - if value is True: - return "true" - if value is False: - return "false" - return str(value) - - -def _requirement(field: FieldSpec) -> str: - parts = ["required" if field.required else "optional"] - if field.has_default: - parts.append(f"default: ``{_literal(field.default)}``") - if field.choices: - choices = ", ".join(f"``{_literal(value)}``" for value in field.choices) - parts.append(f"allowed: {choices}") - if field.legacy: - parts.append("legacy") - return "; ".join(parts) - - -def _field_table(fields: Sequence[FieldSpec]) -> list[str]: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def _field_table( + definition: dict[str, Any], + *, + include: set[str] | None = None, + exclude: set[str] | None = None, +) -> list[str]: + properties = definition.get("properties", {}) + required = set(definition.get("required", [])) + names = [ + name for name in properties + if (include is None or name in include) and (exclude is None or name not in exclude) + ] + if not names: + return ["This variant adds no fields.", ""] # REUSE-IgnoreStart lines = [ - ".. list-table::", + ".. list-table:: Fields", " :header-rows: 1", - " :widths: 18 14 28 40", + " :widths: 18 19 25 38", "", " * - Field", " - Type", " - Requirement", - " - Description", + " - Description and example", ] - for field in fields: + for name in names: + field = properties[name] + requirement = "required" if name in required else "optional" + if "default" in field: + requirement += f"; default ``{_literal(field['default'])}``" + details = field.get("description", "") + examples = field.get("examples", []) + if examples: + details += f" Example: ``{_literal(examples[0])}``." lines.extend(( - f" * - ``{field.name}``", - f" - {_type_name(field)}", - f" - {_requirement(field)}", - f" - {field.description}", + f" * - ``{name}``", + f" - ``{_type_name(field)}``", + f" - {requirement}", + f" - {details}", )) + lines.append("") return lines -def _yaml_block(value: Mapping[str, Any]) -> list[str]: - rendered = yaml.safe_dump( - dict(value), sort_keys=False, default_flow_style=False, allow_unicode=True, - ).rstrip() - return [".. code-block:: yaml", ""] + [f" {line}" for line in rendered.splitlines()] - - -def _mapping_section(title: str, specs: Sequence[MappingSpec], prefix: str) -> list[str]: - lines = [title, "-" * len(title), ""] - for spec in specs: - lines.extend(( - f".. _recipe-{prefix}-{spec.name}:", - "", - f"**``{spec.name}``**", - "", - spec.description, - "", - )) - lines.extend(_field_table(spec.fields)) - lines.extend(("", "Canonical mapping:", "")) - lines.extend(_yaml_block(spec.example)) - lines.append("") +def _model_section( + definition_name: str, + definitions: dict[str, Any], + anchor: str, + *, + include: set[str] | None = None, + exclude: set[str] | None = None, +) -> list[str]: + definition = definitions[definition_name] + title = definition.get("title", definition_name) + lines = [f".. _{anchor}:", "", title, "~" * len(title), ""] + if definition.get("description"): + lines.extend((definition["description"], "")) + lines.extend(_field_table(definition, include=include, exclude=exclude)) return lines -def render_recipe_reference() -> str: - """Render the deterministic standalone recipe-language RST reference.""" - public_steps = tuple(spec for spec in STEP_SPECS if spec.source_allowed) - examples = {spec.name: _canonical_step_example(spec) for spec in public_steps} - for spec in INPUT_MAPPING_SPECS: - _validate_mapping_example(spec, output=False) - for spec in OUTPUT_MAPPING_SPECS: - _validate_mapping_example(spec, output=True) +def _discriminator_mapping( + definitions: dict[str, Any], name: str +) -> dict[str, str]: + definition = definitions[name] + mapping = definition.get("discriminator", {}).get("mapping") + if not isinstance(mapping, dict) or not mapping: + raise ValueError(f"$defs.{name} has no discriminator mapping") + return {key: _reference_name(reference) for key, reference in mapping.items()} + + +def _common_step_fields( + definitions: dict[str, Any], step_names: list[str] +) -> set[str]: + property_sets = [set(definitions[name].get("properties", {})) for name in step_names] + common = set.intersection(*property_sets) + result: set[str] = set() + for field_name in common: + values = [definitions[name]["properties"][field_name] for name in step_names] + if all(value == values[0] for value in values[1:]): + result.add(field_name) + return result + + +def render_reference(schema: dict[str, Any]) -> str: + """Render deterministic RST using only a parsed JSON Schema document.""" + definitions = schema["$defs"] + steps = _discriminator_mapping(definitions, "Step") + inputs = _discriminator_mapping(definitions, "InputMapping") + outputs = _discriminator_mapping(definitions, "OutputMapping") + common_fields = _common_step_fields(definitions, list(steps.values())) lines = [ ".. SPDX-FileCopyrightText: 2026 CERN ", "..", ".. SPDX-License-Identifier: CC-BY-SA-4.0", "..", - ".. This file is generated by pypts.recipe_reference. Do not edit it manually.", - "", - "Recipe Language Reference", - "=========================", + ".. Generated from recipe_language.schema.json. Do not edit manually.", "", - f"Canonical recipe language version: ``{CANONICAL_RECIPE_VERSION}``.", + "Recipe Language 2.0 Reference", + "=============================", "", - "Document grammar", - "----------------", + "This page is generated from the current build's aggregate JSON Schema. It", + "describes", + "the production recipe language accepted by parsing, execution, and YamVIEW.", "", - "A recipe is safe multi-document YAML. The first document is one recipe", - "header and every following document is one sequence. At least one sequence", - "is required, and ``main_sequence`` must name one of them.", + ":download:`Download the JSON Schema `.", "", - "Recipe header", - "-------------", + "See :doc:`/recipe_language_architecture` for parsing, semantic rules,", + "documentation maintenance, and the planned YamVIEW and sequencer flows.", "", - HEADER_SPEC.description, + "Documents", + "---------", "", ] # REUSE-IgnoreEnd - lines.extend(_field_table(HEADER_SPEC.fields)) - lines.extend(("", "Sequence", "--------", "", SEQUENCE_SPEC.description, "")) - lines.extend(_field_table(SEQUENCE_SPEC.fields)) - lines.extend(( - "", - "Common step fields", - "------------------", - "", - "These fields are shared by every authorable step type.", - "", - )) - lines.extend(_field_table(COMMON_STEP_FIELDS)) - lines.extend(("", "Registered step types", "---------------------", "")) - common_names = {field.name for field in COMMON_STEP_FIELDS} - for spec in public_steps: - lines.extend(( - f".. _recipe-step-{spec.name.lower()}:", - "", - spec.name, - "~" * len(spec.name), - "", - spec.description, - "", - )) - specific_fields = tuple(field for field in spec.fields if field.name not in common_names) - if specific_fields: - lines.extend(_field_table(specific_fields)) - lines.append("") - if spec.required_inputs: - required = ", ".join(f"``{name}``" for name in spec.required_inputs) - lines.extend((f"Required input names: {required}.", "")) - lines.extend(("Canonical example:", "")) - lines.extend(_yaml_block(examples[spec.name])) - lines.append("") - lines.extend(_mapping_section("Input mapping types", INPUT_MAPPING_SPECS, "input")) - lines.extend(_mapping_section("Output mapping types", OUTPUT_MAPPING_SPECS, "output")) - lines.extend(("Semantic constraints", "--------------------", "")) - for constraint in CONSTRAINT_SPECS: - diagnostics = ", ".join(f"``{code}``" for code in constraint.diagnostic_codes) - lines.extend(( - f"* ``{constraint.scope}`` — {constraint.description}", - f" Diagnostics: {diagnostics}.", + lines.extend(_model_section("RecipeHeader", definitions, "recipe-v2-header")) + lines.extend(_model_section("Sequence", definitions, "recipe-v2-sequence")) + + lines.extend(("Nested structures", "-----------------", "")) + for name in ("InternalSequenceReference", "FileDestination", "UploadFile"): + lines.extend(_model_section(name, definitions, f"recipe-v2-structure-{name.lower()}")) + + lines.extend(("Common step fields", "------------------", "")) + representative = next(iter(steps.values())) + lines.extend(_field_table(definitions[representative], include=common_fields)) + + lines.extend(("Authorable steps", "----------------", "")) + for discriminator, definition_name in steps.items(): + lines.extend(_model_section( + definition_name, + definitions, + f"recipe-v2-step-{discriminator.lower()}", + exclude=common_fields, )) - lines.extend(( - "", - "Canonicalization", - "----------------", - "", - "The parser normalizes step type casing, implicit direct inputs, mapping", - "defaults, and optional flags. Canonical serialization uses explicit YAML", - "document starts and stable field ordering. Comments and original formatting", - "are not preserved; parse/dump/reparse model equality is the guarantee.", - "", - "``IndexedStep`` is reserved for runtime construction and cannot be authored", - "as a recipe step.", - "", - )) - return "\n".join(lines) - - -def write_recipe_reference(path: str | Path = DEFAULT_REFERENCE_PATH) -> None: - """Write the generated reference, creating its parent directory.""" - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(render_recipe_reference(), encoding="utf-8") - - -def check_recipe_reference(path: str | Path = DEFAULT_REFERENCE_PATH) -> bool: - """Return whether an existing reference exactly matches generated output.""" - destination = Path(path) - try: - current = destination.read_text(encoding="utf-8") - except (OSError, UnicodeError): - return False - return current == render_recipe_reference() + lines.extend(("Input mappings", "--------------", "")) + for discriminator, definition_name in inputs.items(): + lines.extend(_model_section( + definition_name, definitions, f"recipe-v2-input-{discriminator}" + )) -def main(argv: Sequence[str] | None = None) -> int: - """Generate or check the standalone reference artifact.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("path", nargs="?", type=Path, default=DEFAULT_REFERENCE_PATH) - parser.add_argument("--check", action="store_true", help="fail if the artifact is missing or stale") - arguments = parser.parse_args(argv) - if arguments.check: - if check_recipe_reference(arguments.path): - return 0 - print(f"Recipe language reference is missing or stale: {arguments.path}", file=sys.stderr) - return 1 - try: - write_recipe_reference(arguments.path) - except OSError as error: - print(f"Could not write recipe language reference: {error}", file=sys.stderr) - return 2 - print(f"Wrote recipe language reference: {arguments.path}") - return 0 + lines.extend(("Output mappings", "---------------", "")) + for discriminator, definition_name in outputs.items(): + lines.extend(_model_section( + definition_name, definitions, f"recipe-v2-output-{discriminator}" + )) + return "\n".join(lines).rstrip() + "\n" -if __name__ == "__main__": - raise SystemExit(main()) +def render_reference_file(path: str | Path) -> str: + return render_reference(load_schema(path)) diff --git a/src/pypts/steps.py b/src/pypts/steps.py index 6635bed..4a3b5e8 100644 --- a/src/pypts/steps.py +++ b/src/pypts/steps.py @@ -1199,6 +1199,8 @@ def __init__( ): super().__init__(continue_on_error=continue_on_error, **kwargs) self.files = files + if permissions is None: + permissions = 0o755 if isinstance(permissions, str): self.permissions = int(permissions, 8) if permissions.startswith("0") else int(permissions) else: diff --git a/tests/functional_tests/test_recipes_format.py b/tests/functional_tests/test_recipes_format.py index bc3b231..1c83116 100644 --- a/tests/functional_tests/test_recipes_format.py +++ b/tests/functional_tests/test_recipes_format.py @@ -2,9 +2,10 @@ # # SPDX-License-Identifier: LGPL-2.1-or-later -from pypts.YamVIEW.verify_recipe import validate_all_recipes_in_folder from pypts.utils import get_project_root +from pypts.YamVIEW.verify_recipe import validate_all_recipes_in_folder + def test_recipes_format(): recipe_path = get_project_root() / "src" / "pypts" / "recipes" - assert validate_all_recipes_in_folder(recipe_path) \ No newline at end of file + assert not validate_all_recipes_in_folder(recipe_path) diff --git a/tests/unit_tests/test_a_gui.py b/tests/unit_tests/test_a_gui.py index 446859b..285f46c 100644 --- a/tests/unit_tests/test_a_gui.py +++ b/tests/unit_tests/test_a_gui.py @@ -12,6 +12,7 @@ from PySide6.QtCore import Qt from pypts import gui, recipe +from pypts.recipe_language import Sequence as SequenceDefinition from pypts.startup import create_and_start_gui from pypts.gui_components import interaction_panel from pypts.gui_components.results_panel import StepResultModel @@ -31,15 +32,16 @@ def sample_sequence(): step1 = recipe.Step(step_name="Test Step 1", description="First step") step2 = recipe.Step(step_name="Test Step 2", description="Second step") sequence = recipe.Sequence( - sequence_data={ + SequenceDefinition.model_validate({ "sequence_name": "Test Sequence", + "description": "GUI fixture.", "locals": {}, "parameters": {}, "outputs": {}, "setup_steps": [], "steps": [], "teardown_steps": [], - } + }) ) sequence.steps = [step1, step2] return sequence @@ -233,15 +235,16 @@ def test_on_start_clicked_switches_to_running_tab_before_queue_start(main_window def fake_load_recipe(): load_states.append(main_window._screen_idx) sequence = recipe.Sequence( - sequence_data={ + SequenceDefinition.model_validate({ "sequence_name": "Test Sequence", + "description": "GUI fixture.", "locals": {}, "parameters": {}, "outputs": {}, "setup_steps": [], "steps": [], "teardown_steps": [], - } + }) ) sequence.steps = [recipe.Step(step_name="Step 1", description="First")] main_window.recipe_to_run = Mock(sequences={"main": sequence}, main_sequence="main") diff --git a/tests/unit_tests/test_recipe.py b/tests/unit_tests/test_recipe.py index 7180387..e3abee6 100644 --- a/tests/unit_tests/test_recipe.py +++ b/tests/unit_tests/test_recipe.py @@ -1,872 +1,203 @@ -# SPDX-FileCopyrightText: 2025 CERN -# +# SPDX-FileCopyrightText: 2026 CERN # SPDX-License-Identifier: LGPL-2.1-or-later +"""Typed recipe-to-runtime construction and execution tests.""" -"""Tests for pypts.recipe — covers ResultType enum, Recipe loading, -Sequence construction/execution, Step, StepResult, Runtime, and serialize().""" +import queue import pytest -import uuid -import queue -from enum import Enum -from pathlib import Path -from threading import Event -from unittest.mock import MagicMock, patch +from pydantic import TypeAdapter import pypts.recipe from pypts.recipe import ( + STEP_TYPE_REGISTRY, + IndexedStep, Recipe, + ResultType, Runtime, - Sequence, Step, - IndexedStep, - PythonModuleStep, - SequenceStep, - UserInteractionStep, - WaitStep, - SerialNumberStep, - StepResult, - ResultType, - serialize, ) - - -# ============================================================ -# Helpers -# ============================================================ - -def _make_recipe_data(overrides=None, sequences=None): - """Build minimal valid recipe data for the file_loader.""" - main = { - "name": "Test", - "description": "desc", - "version": "1.0", - "main_sequence": "Main", - "globals": {}, +from pypts.recipe_language import STEP_MODELS +from pypts.recipe_language import Recipe as RecipeDefinition +from pypts.recipe_language import Step as AuthorableStep +from pypts.recipe_parser import RecipeParseError, dump_recipe + + +def step_example(kind, **updates): + value = { + "steptype": kind, + "step_name": kind, + "description": f"Exercise {kind}.", + "input_mapping": {}, + "output_mapping": {}, + } + value.update(updates) + return value + + +STEP_EXAMPLES = { + "PythonModuleStep": step_example( + "PythonModuleStep", action_type="method", module="tests.py", method_name="run" + ), + "SequenceStep": step_example( + "SequenceStep", sequence={"type": "internal", "name": "Target"} + ), + "UserInteractionStep": step_example("UserInteractionStep"), + "WaitStep": step_example( + "WaitStep", input_mapping={"wait_time": {"type": "direct", "value": 0}} + ), + "UserLoadingStep": step_example( + "UserLoadingStep", + file_save_location={"type": "local", "variable": "selected"}, + ), + "UserRunMethodStep": step_example( + "UserRunMethodStep", + trigger_response="run", + action_type="method", + module="tests.py", + method_name="run", + ), + "UserWriteStep": step_example("UserWriteStep"), + "SerialNumberStep": step_example("SerialNumberStep"), + "SSHConnectStep": step_example("SSHConnectStep"), + "SSHCloseStep": step_example("SSHCloseStep"), + "SSHUploadStep": step_example( + "SSHUploadStep", + files=[{"local": "bin/tool", "remote": "/tmp/tool"}], + permissions="0755", + skip_if_sha256_match=True, + local_package="fixtures", + ), +} + + +def wait_step(name, *, indexed=False): + value = [0, 0] if indexed else 0 + mapping = {"type": "direct", "value": value} + if indexed: + mapping["indexed"] = True + return { + "steptype": "WaitStep", + "step_name": name, + "description": name, + "input_mapping": {"wait_time": mapping}, } - if overrides: - main.update(overrides) - - seq = sequences or [{ - "sequence_name": "Main", - "locals": {}, - "parameters": {}, - "outputs": {}, - "setup_steps": [], - "steps": [], - "teardown_steps": [], - }] - return [main] + seq - - -def _loader_for(data): - """Create a mock file_loader that yields the given data.""" - loader = MagicMock() - loader.return_value = iter(data) - return loader - - -# ============================================================ -# Fixtures -# ============================================================ - -@pytest.fixture -def runtime(): - """Create a Runtime instance with clean class-level state.""" - Runtime.stop_event.clear() - Runtime.recipe_thread = None - Runtime.recipe_event_proxy = None - eq = queue.SimpleQueue() - rq = queue.SimpleQueue() - yield Runtime(eq, rq) - Runtime.stop_event.clear() - Runtime.recipe_thread = None - Runtime.recipe_event_proxy = None - - -@pytest.fixture -def recipe_from_yaml(): - """Load the real test_recipe.yaml file into a Recipe object.""" - recipe_file_path = Path(__file__).parent / "test_recipe.yaml" - return Recipe(recipe_file_path) - - -# ============================================================ -# ResultType enum -# ============================================================ - -class TestResultType: - def test_enum_values(self): - """Verify the integer values assigned to each ResultType member.""" - assert ResultType.SKIP.value == 0 - assert ResultType.DONE.value == 1 - assert ResultType.PASS.value == 2 - assert ResultType.FAIL.value == 3 - assert ResultType.ERROR.value == 4 - - def test_str_representation(self): - """Verify that str(ResultType.X) returns the member name.""" - assert str(ResultType.SKIP) == "SKIP" - assert str(ResultType.DONE) == "DONE" - assert str(ResultType.PASS) == "PASS" - assert str(ResultType.FAIL) == "FAIL" - assert str(ResultType.ERROR) == "ERROR" - - def test_enum_consistency(self): - """Verify that str() matches .name for every member.""" - for result_type in ResultType: - assert str(result_type) == result_type.name - - def test_access_by_name(self): - """Verify bracket access by string name (ResultType['PASS']).""" - assert ResultType["SKIP"] == ResultType.SKIP - assert ResultType["DONE"] == ResultType.DONE - assert ResultType["PASS"] == ResultType.PASS - assert ResultType["FAIL"] == ResultType.FAIL - assert ResultType["ERROR"] == ResultType.ERROR - - def test_invalid_name_raises_key_error(self): - """Verify that accessing a non-existent member by name raises KeyError.""" - with pytest.raises(KeyError): - ResultType["INVALID"] - - def test_severity_ordering(self): - """Verify severity ordering: SKIP < DONE < PASS < FAIL < ERROR < STOP.""" - assert ResultType.SKIP < ResultType.DONE < ResultType.PASS < ResultType.FAIL < ResultType.ERROR < ResultType.STOP - - def test_all_members_have_str(self): - """Verify that every ResultType member has a valid string representation.""" - for member in ResultType: - assert str(member) == member.name - - -# ============================================================ -# Recipe loading (mock file_loader) -# ============================================================ - -class TestRecipeLoading: - def test_loads_valid_recipe(self): - """Verify basic recipe loading sets name, version, and sequences.""" - data = _make_recipe_data() - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.name == "Test" - assert r.version == "1.0" - assert "Main" in r.sequences - - def test_missing_main_sequence_defaults_to_main(self): - data = _make_recipe_data() - del data[0]["main_sequence"] - assert Recipe("fake.yaml", file_loader=_loader_for(data)).main_sequence == "Main" - - def test_unknown_main_sequence_raises_descriptive_error(self): - data = _make_recipe_data(overrides={"main_sequence": "Missing"}) - with pytest.raises(ValueError, match="Main sequence 'Missing' does not exist"): - Recipe("fake.yaml", file_loader=_loader_for(data)) - - def test_loading_with_event_sender(self): - """Verify that running a loaded recipe emits pre_run_recipe via the event sender.""" - recipe_data = [ - {"name": "Test Recipe", "description": "For testing", "version": "1.0", - "main_sequence": "Main", "globals": {}}, - {"sequence_name": "Main", "locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": []} - ] - - mock_file_loader = MagicMock() - mock_file_loader.return_value = iter(recipe_data) - - sent_events = [] - def mock_event_sender(runtime, event_name, *event_data): - sent_events.append((runtime, event_name, event_data)) - - r = Recipe( - recipe_file_path="fake_path.yaml", - file_loader=mock_file_loader, - event_sender=mock_event_sender - ) - - assert r.name == "Test Recipe" - assert r.description == "For testing" - assert "Main" in r.sequences - - mock_runtime = MagicMock() - r.run(runtime=mock_runtime) - - assert len(sent_events) > 0 - assert sent_events[0][1] == "pre_run_recipe" - assert sent_events[0][2] == ("Test Recipe", "For testing") - - def test_missing_required_field_raises(self): - """Verify that omitting a required field (name/description/version/globals) raises.""" - for field in ["name", "description", "version", "globals"]: - data = _make_recipe_data() - del data[0][field] - with pytest.raises(Exception): - Recipe("fake.yaml", file_loader=_loader_for(data)) - - def test_report_overwrite_mode(self): - """Verify that report='overwrite' sets report_overwrite to True.""" - data = _make_recipe_data(overrides={"report": "overwrite"}) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.report_overwrite is True - - def test_report_append_mode(self): - """Verify that report='append' sets report_overwrite to False.""" - data = _make_recipe_data(overrides={"report": "append"}) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.report_overwrite is False - - def test_report_name_include_serial_flag(self): - """Verify report_name_include_serial is parsed from recipe main section.""" - data = _make_recipe_data(overrides={"report_name_include_serial": True}) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.report_name_include_serial is True - - def test_invalid_report_mode_raises(self): - """Verify that an unsupported report mode raises an exception.""" - data = _make_recipe_data(overrides={"report": "invalid_mode"}) - with pytest.raises(Exception): - Recipe("fake.yaml", file_loader=_loader_for(data)) - - def test_dotted_test_package_is_supported(self): - data = _make_recipe_data(overrides={"test_package": "my.package"}) - assert Recipe("fake.yaml", file_loader=_loader_for(data)).test_package == "my.package" - - @pytest.mark.parametrize("package", ["bad-name", ".leading", "trailing.", "two..dots", "1package"]) - def test_invalid_test_package_raises_descriptive_error(self, package): - data = _make_recipe_data(overrides={"test_package": package}) - with pytest.raises(ValueError, match="valid dotted Python package name"): - Recipe("fake.yaml", file_loader=_loader_for(data)) - - def test_test_package_none_ok(self): - """Verify that omitting test_package results in None.""" - data = _make_recipe_data() - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.test_package is None - - def test_multiple_sequences(self): - """Verify that multiple sequences are loaded correctly.""" - seqs = [ - {"sequence_name": "Main", "locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": []}, - {"sequence_name": "Sub", "locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": []}, - ] - data = _make_recipe_data(sequences=seqs) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert "Main" in r.sequences - assert "Sub" in r.sequences - - def test_sequence_missing_name_skipped(self): - """Verify that a sequence document without 'sequence_name' is silently skipped.""" - seqs = [ - {"sequence_name": "Main", "locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": []}, - {"locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": []}, - ] - data = _make_recipe_data(sequences=seqs) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert len(r.sequences) == 1 - - def test_recipe_file_name_stored(self): - """Verify that recipe_file_name stores only the filename (not the full path).""" - data = _make_recipe_data() - r = Recipe("path/to/my_recipe.yaml", file_loader=_loader_for(data)) - assert r.recipe_file_name == "my_recipe.yaml" - - def test_globals_stored(self): - """Verify that global variables from the recipe are stored.""" - data = _make_recipe_data(overrides={"globals": {"host": "10.0.0.1", "port": 22}}) - r = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert r.globals == {"host": "10.0.0.1", "port": 22} - - def test_top_level_continue_on_error_is_stored_as_recipe_policy(self): - data = _make_recipe_data(overrides={"continue_on_error": True}) - recipe = Recipe("fake.yaml", file_loader=_loader_for(data)) - assert recipe.continue_on_error is True - - def test_top_level_continue_on_error_must_be_boolean(self): - data = _make_recipe_data(overrides={"continue_on_error": "true"}) - with pytest.raises(ValueError, match="must be a boolean"): - Recipe("fake.yaml", file_loader=_loader_for(data)) - - @pytest.mark.parametrize( - ("selection", "expected"), [(None, "Main"), ("Alternate", "Alternate")] - ) - def test_run_uses_default_or_explicit_sequence(self, runtime, selection, expected): - sequences = _make_recipe_data()[1:] + [{ - "sequence_name": "Alternate", "locals": {}, "parameters": {}, "outputs": {}, - "setup_steps": [], "steps": [], "teardown_steps": [], - }] - recipe = Recipe("fake.yaml", file_loader=_loader_for(_make_recipe_data(sequences=sequences))) - built_step = MagicMock() - built_step.run.return_value = MagicMock() - with patch("pypts.recipe.Step.build_step", return_value=built_step) as builder, \ - patch("pypts.recipe.time.sleep"): - recipe.run(runtime, sequence_name=selection) - assert builder.call_args.args[0]["sequence"]["name"] == expected - - def test_run_rejects_unknown_explicit_sequence(self, runtime): - recipe = Recipe("fake.yaml", file_loader=_loader_for(_make_recipe_data())) - with pytest.raises(ValueError, match="Sequence 'Missing' does not exist"), \ - patch("pypts.recipe.time.sleep"): - recipe.run(runtime, sequence_name="Missing") - - -# ============================================================ -# Recipe from YAML file -# ============================================================ - -class TestRecipeFromYaml: - def test_metadata_loaded_correctly(self, recipe_from_yaml): - """Verify that the YAML recipe name, version, and description are parsed.""" - assert recipe_from_yaml.name == "Example Test Recipe" - assert recipe_from_yaml.version == "0.1.0" - assert recipe_from_yaml.description.startswith("A sample recipe") - - def test_main_sequence_exists_and_has_steps(self, recipe_from_yaml): - """Verify the Main sequence has the expected number of steps.""" - assert "Main" in recipe_from_yaml.sequences - main_sequence = recipe_from_yaml.sequences["Main"] - assert hasattr(main_sequence, "steps") - assert isinstance(main_sequence.steps, list) - assert len(main_sequence.steps) == 5 - - def test_range_check_pass_fail(self, recipe_from_yaml, runtime): - """Verify that a value inside range produces PASS and outside range produces FAIL.""" - runtime.set_globals(recipe_from_yaml.globals) - runtime.set_sequences(recipe_from_yaml.sequences) - runtime.test_package = recipe_from_yaml.test_package - - inside_step = recipe_from_yaml.sequences["Main"].steps[3] - outside_step = recipe_from_yaml.sequences["Main"].steps[4] - - inside_result = inside_step.process_outputs(runtime, inside_step.run(runtime, {}).outputs) - outside_result = outside_step.process_outputs(runtime, outside_step.run(runtime, {}).outputs) - - assert inside_result is ResultType.PASS - assert outside_result is ResultType.FAIL - - -# ============================================================ -# Sequence -# ============================================================ - -class TestSequence: - def test_no_data_no_file_raises(self): - """Verify that creating a Sequence with no data and no file raises FileNotFoundError.""" - with pytest.raises(FileNotFoundError): - Sequence() - - def test_from_data(self): - """Verify that a Sequence can be constructed from a data dict.""" - seq_data = { - "sequence_name": "TestSeq", - "locals": {"x": 1}, - "parameters": {"p1": {}}, - "outputs": {"o1": {}}, - "setup_steps": [], - "steps": [{"steptype": "WaitStep", "step_name": "W", "input_mapping": {}, "output_mapping": {}}], - "teardown_steps": [], - } - seq = Sequence(sequence_data=seq_data) - assert seq.name == "TestSeq" - assert seq.locals == {"x": 1} - assert len(seq.steps) == 1 - - def test_from_data_with_build_step_mock(self): - """Verify Sequence construction with mocked Step.build_step.""" - mock_step = MagicMock() - mock_step.name = "MockStep" - - sequence_data = { - "sequence_name": "TestSeq", - "locals": {"var1": None}, - "parameters": {}, - "outputs": {}, - "setup_steps": [], - "steps": [{"steptype": "DummyStep"}], - "teardown_steps": [] - } - - with patch("pypts.recipe.Step.build_step", return_value=mock_step): - sequence = Sequence(sequence_data=sequence_data) - - assert sequence.name == "TestSeq" - assert sequence.locals == {"var1": None} - assert len(sequence.steps) == 1 - assert sequence.steps[0].name == "MockStep" - - def test_run_executes_steps(self): - """Verify that Sequence.run calls setup, main, and teardown steps via Step.run_steps.""" - mock_runtime = MagicMock() - mock_runtime.send_event = MagicMock() - mock_runtime.set_local = MagicMock() - mock_runtime.push_locals = MagicMock() - mock_runtime.pop_locals = MagicMock() - mock_step_result = MagicMock() - with patch("pypts.recipe.StepResult.evaluate_multiple_step_results", return_value="final_result"), \ - patch("pypts.recipe.Step.run_steps", - side_effect=lambda runtime, steps, parent, **kwargs: [mock_step_result] * len(steps)): - sequence_data = { - "sequence_name": "TestRun", - "locals": {}, +def definition(): + return RecipeDefinition.model_validate({ + "header": { + "name": "Runtime recipe", + "version": "1.0", + "recipe_version": "2.0.0", + "description": "Typed runtime fixture.", + "main_sequence": "Main", + "globals": {"shared": 1}, + }, + "sequences": [ + { + "sequence_name": "Main", + "description": "Main.", "parameters": {}, "outputs": {}, - "setup_steps": [], - "steps": [{"steptype": "DummyStep"}], - "teardown_steps": [{"steptype": "TeardownStep"}] - } - - with patch("pypts.recipe.Step.build_step", return_value=MagicMock()): - seq = Sequence(sequence_data=sequence_data) - result = seq.run(mock_runtime, input={"test_param": 123}) - - assert result == "final_result" - assert mock_runtime.set_local.called - assert mock_runtime.push_locals.called - assert mock_runtime.pop_locals.called - assert mock_runtime.send_event.call_count == 2 # pre_run_sequence + post_run_sequence - - def test_teardown_runs_even_after_error(self): - """Verify that teardown steps execute even when main steps raise an exception.""" - mock_runtime = MagicMock() - mock_runtime.stop_event = Event() - mock_runtime.send_event = MagicMock() - mock_runtime.push_locals = MagicMock() - mock_runtime.pop_locals = MagicMock() - mock_runtime.set_local = MagicMock() - - mock_teardown_result = MagicMock() - mock_teardown_result.get_result.return_value = ResultType.DONE - - call_count = [0] - def run_steps_side_effect(runtime, steps, parent, **kwargs): - call_count[0] += 1 - if call_count[0] == 1: - raise ValueError("step failure") - return [mock_teardown_result] - - with patch("pypts.recipe.Step.build_step", return_value=MagicMock()), \ - patch("pypts.recipe.Step.run_steps", side_effect=run_steps_side_effect), \ - patch("pypts.recipe.StepResult.evaluate_multiple_step_results", return_value=ResultType.ERROR): - seq_data = { - "sequence_name": "S", - "locals": {}, + "locals": {"local": 1}, + "setup_steps": [wait_step("setup")], + "steps": [ + wait_step("indexed", indexed=True), + { + "steptype": "SequenceStep", + "step_name": "nested", + "description": "Nested.", + "sequence": {"type": "internal", "name": "Sub"}, + }, + ], + "teardown_steps": [wait_step("teardown")], + }, + { + "sequence_name": "Sub", + "description": "Sub.", "parameters": {}, "outputs": {}, + "locals": {}, "setup_steps": [], - "steps": [{"steptype": "DummyStep"}], - "teardown_steps": [{"steptype": "DummyStep"}], - } - seq = Sequence(sequence_data=seq_data) - result = seq.run(mock_runtime, {}) - - # Teardown was called (run_steps called twice total) - assert call_count[0] == 2 - - -# ============================================================ -# Step -# ============================================================ - -class TestStep: - def test_initialization(self): - """Verify Step stores name and generates a UUID.""" - step = Step(step_name="MyStep", input_mapping={"a": {"type": "direct", "value": 1}}) - assert step.name == "MyStep" - assert isinstance(step.id, uuid.UUID) - - def test_check_indexing_false(self): - """Verify check_indexing returns False when no input is marked indexed.""" - step = Step("TestStep", input_mapping={"a": {"type": "direct", "value": 5}}) - assert step.check_indexing() is False - - def test_check_indexing_true(self): - """Verify check_indexing returns True when an input has indexed=True.""" - step = Step("TestStep", input_mapping={"a": {"type": "direct", "value": 5, "indexed": True}}) - assert step.check_indexing() is True - - def test_image_output_accepts_multiple_paths(self): - """Verify one image output key can provide multiple figure paths.""" - step = Step( - step_name="ImageStep", - input_mapping={}, - output_mapping={"chart": {"type": "image"}}, - ) - - runtime = MagicMock() - runtime.recipe_name = "R" - runtime.recipe_file_name = "r.yml" - runtime.serial_number = "SN" - runtime.current_sequence_name = "Main" - runtime.pypts_version = "test" - runtime.stop_event = Event() - runtime.report_queue = queue.SimpleQueue() - - step._step = lambda runtime, step_input, parent_uuid: { - "chart": ["plot_a.png", "plot_b.png"], - } - - result = step.run(runtime, {}) - assert result.image_paths == ["plot_a.png", "plot_b.png"] - - -# ============================================================ -# SerialNumberStep -# ============================================================ - -class TestSerialNumberStep: - def test_sends_event_and_stores_serial(self): - """Verify SerialNumberStep sends get_serial_number event, stores the serial - in runtime.serial_number, and sets the global variable.""" - step = SerialNumberStep( - step_name="Get Serial Number", - input_mapping={}, - output_mapping={} - ) - - mock_runtime = MagicMock() - mock_runtime.stop_event = Event() - - def mock_send_event(event_name, response_q, *args): - if event_name == "get_serial_number": - response_q.put("TEST123") - - mock_runtime.send_event.side_effect = mock_send_event - - result = step._step(mock_runtime, {}, uuid.uuid4()) - - assert result["serial_number"] == "TEST123" - assert mock_runtime.serial_number == "TEST123" - mock_runtime.set_global.assert_called_once_with("serial_number", "TEST123") - - -# ============================================================ -# StepResult -# ============================================================ - -class TestStepResult: - def test_initialization_defaults(self): - """Verify StepResult defaults: UUID generated, result is None, empty inputs/outputs.""" - sr = StepResult() - assert isinstance(sr.uuid, uuid.UUID) - assert sr.result is None - assert sr.inputs == {} - assert sr.outputs == {} - assert sr.error_info == "" - assert isinstance(sr.subresults, list) - assert sr.parent is None - assert sr.recipe_name is None - assert sr.recipe_file_name is None - assert sr.sequence_name is None - assert sr.pypts_version == "unknown" - - def test_set_error(self): - """Verify set_error sets result to ERROR with error_info and inputs.""" - sr = StepResult() - sr.set_error(error_info="An error occurred", inputs={"key": "value"}) - assert sr.result == ResultType.ERROR - assert sr.error_info == "An error occurred" - assert sr.inputs == {"key": "value"} - - def test_set_skip(self): - """Verify set_skip sets result to SKIP.""" - sr = StepResult() - sr.set_skip() - assert sr.result == ResultType.SKIP - - def test_set_result(self): - """Verify set_result stores the given result type, inputs, and outputs.""" - sr = StepResult() - sr.set_result(result_type=ResultType.PASS, inputs={"input1": "value"}, outputs={"output1": "value"}) - assert sr.result == ResultType.PASS - assert sr.inputs == {"input1": "value"} - assert sr.outputs == {"output1": "value"} - - def test_set_stop(self): - """Verify set_stop sets result to STOP with error_info and inputs.""" - sr = StepResult() - sr.set_stop(error_info="aborted", inputs={"k": "v"}) - assert sr.result == ResultType.STOP - assert sr.error_info == "aborted" - assert sr.inputs == {"k": "v"} - - def test_append_subresult(self): - """Verify that subresults can be appended and are stored correctly.""" - sr = StepResult() - sub = StepResult() - sub.set_result(ResultType.FAIL) - sr.append_subresult(sub) - assert len(sr.subresults) == 1 - assert sr.subresults[0].result == ResultType.FAIL - - def test_get_result_by_uuid(self): - """Verify that a StepResult can be found by its UUID.""" - sr1 = StepResult() - sr1.set_result(ResultType.PASS) - sr2 = StepResult() - sr2.set_result(ResultType.FAIL) - - found = StepResult.get_result_by_uuid([sr1, sr2], sr2.uuid) - assert found.result == ResultType.FAIL - - def test_get_result_by_uuid_not_found(self): - """Verify that searching for a non-existent UUID returns None.""" - sr = StepResult() - assert StepResult.get_result_by_uuid([sr], uuid.uuid4()) is None + "steps": [wait_step("sub")], + "teardown_steps": [], + }, + ], + }) - def test_get_result_by_uuid_in_subresults(self): - """Verify that searching finds results nested in subresults.""" - parent = StepResult() - child = StepResult() - parent.append_subresult(child) - found = StepResult.get_result_by_uuid([parent], child.uuid) - assert found is child - def test_evaluate_multiple_step_results(self): - """Verify that evaluating multiple results returns the highest severity.""" - sr1 = StepResult() - sr1.set_result(ResultType.PASS) - sr2 = StepResult() - sr2.set_result(ResultType.FAIL) - - highest = StepResult.evaluate_multiple_step_results([sr1, sr2]) - assert highest == ResultType.FAIL # FAIL > PASS - - def test_evaluate_all_skip(self): - """Verify that when all results are SKIP, evaluation returns SKIP.""" - results = [StepResult() for _ in range(3)] - for r in results: - r.set_skip() - assert StepResult.evaluate_multiple_step_results(results) == ResultType.SKIP - - def test_evaluate_mixed(self): - """Verify that ERROR is returned when mixed with DONE and PASS.""" - r1 = StepResult() - r1.set_result(ResultType.DONE) - r2 = StepResult() - r2.set_result(ResultType.ERROR) - r3 = StepResult() - r3.set_result(ResultType.PASS) - assert StepResult.evaluate_multiple_step_results([r1, r2, r3]) == ResultType.ERROR - - def test_is_type(self): - """Verify is_type returns True for matching type and False otherwise.""" - sr = StepResult() - sr.set_result(ResultType.PASS) - assert sr.is_type(ResultType.PASS) - assert not sr.is_type(ResultType.FAIL) - - def test_print_result(self): - """Verify set_result(PASS) can be set without raising exceptions.""" - sr = StepResult() - sr.set_result(ResultType.PASS) - assert sr.result == ResultType.PASS - - def test_print_result_with_subresults(self): - """Verify subresults are accessible after being appended.""" - sr = StepResult() - sr.set_result(ResultType.PASS) - sub = StepResult() - sub.set_result(ResultType.FAIL) - sr.append_subresult(sub) - assert len(sr.subresults) == 1 - assert sr.subresults[0].result == ResultType.FAIL - - -# ============================================================ -# serialize() -# ============================================================ - -class Color(Enum): - RED = 1 - GREEN = 2 - BLUE = 3 - - -class TestSerialize: - def test_enum(self): - """Verify that Enum members serialize to their name.""" - assert serialize(Color.RED) == "RED" - assert serialize(Color.GREEN) == "GREEN" - assert serialize(Color.BLUE) == "BLUE" - - def test_result_type_enum(self): - """Verify that ResultType members serialize to their name.""" - assert serialize(ResultType.PASS) == "PASS" - - def test_basic_types(self): - """Verify that int, float, and str serialize to their string representation.""" - assert serialize(1234) == "1234" - assert serialize(45.67) == "45.67" - assert serialize("hello") == "hello" - - def test_none(self): - """Verify that None serializes to the string 'None'.""" - assert serialize(None) == "None" - - def test_dict(self): - """Verify that dicts are serialized recursively.""" - result = serialize({"a": 1, "b": ResultType.FAIL}) - assert result == {"a": "1", "b": "FAIL"} - - def test_list(self): - """Verify that lists are serialized recursively.""" - result = serialize([1, "hello", ResultType.DONE]) - assert result == ["1", "hello", "DONE"] - - def test_set(self): - """Verify that sets are serialized to a list.""" - result = serialize({1}) - assert isinstance(result, list) - assert "1" in result - - def test_tuple(self): - """Verify that tuples are serialized to a list.""" - result = serialize((1, 2)) - assert result == ["1", "2"] - - def test_object_with_dict(self): - """Verify that objects with __dict__ are serialized recursively.""" - class Obj: - def __init__(self): - self.x = 42 - result = serialize(Obj()) - assert result["x"] == "42" - - def test_circular_reference(self): - """Verify that circular references are handled without infinite recursion.""" - d = {} - d["self"] = d - result = serialize(d) - assert "Circular reference" in str(result) - - -# ============================================================ -# Runtime -# ============================================================ - -class TestRuntime: - def test_initialization(self, runtime): - """Verify Runtime default state after construction.""" - assert isinstance(runtime, Runtime) - assert runtime.event_queue.empty() - assert runtime.report_queue.empty() - assert runtime.results == [] - assert runtime.globals == [] - assert runtime.sequences == {} - assert runtime.local_stack == [] - assert runtime.recipe_name is None - assert runtime.recipe_file_name is None - assert runtime.current_sequence_name is None - assert runtime.pypts_version == "unknown" - - def test_push_locals(self, runtime): - """Verify pushing locals adds them to the stack.""" - runtime.push_locals({'a': 1, 'b': 2}) - assert len(runtime.local_stack) == 1 - assert runtime.local_stack[-1] == {'a': 1, 'b': 2} - - def test_pop_locals(self, runtime): - """Verify popping locals removes and returns the top frame.""" - runtime.push_locals({'a': 1, 'b': 2}) - popped = runtime.pop_locals() - assert popped == {'a': 1, 'b': 2} - assert len(runtime.local_stack) == 0 - - def test_get_local(self, runtime): - """Verify that get_local retrieves values from the top locals frame.""" - runtime.push_locals({'a': 1, 'b': 2}) - assert runtime.get_local('a') == 1 - assert runtime.get_local('b') == 2 - - def test_set_local(self, runtime): - """Verify that set_local updates a value in the top locals frame.""" - runtime.push_locals({'a': 1, 'b': 2}) - runtime.set_local('a', 10) - assert runtime.get_local('a') == 10 - assert runtime.get_local('b') == 2 - - def test_local_stack_multiple_levels(self, runtime): - """Verify that multiple push/pop operations maintain correct stack semantics.""" - runtime.push_locals({"a": 1}) - runtime.push_locals({"b": 2}) - assert runtime.get_local("b") == 2 - runtime.pop_locals() - assert runtime.get_local("a") == 1 - - def test_get_local_missing_raises(self, runtime): - """Verify that accessing a non-existent local variable raises KeyError.""" - runtime.push_locals({"a": 1}) - with pytest.raises(KeyError): - runtime.get_local("nonexistent") - - def test_set_globals(self, runtime): - """Verify setting and getting global variables.""" - globals_data = {'global1': 'value1', 'global2': 'value2'} - runtime.set_globals(globals_data) - assert runtime.get_globals() == globals_data - - def test_set_global_and_get(self, runtime): - """Verify individual global variable access.""" - runtime.set_globals({"x": 10, "y": 20}) - assert runtime.get_global("x") == 10 - assert runtime.get_global("y") == 20 - - def test_get_global_missing_returns_none(self, runtime): - """Verify that accessing a non-existent global returns None.""" - runtime.set_globals([]) - assert runtime.get_global(99) is None - - def test_sequences(self, runtime): - """Verify setting and getting sequences.""" - mock_seq = MagicMock() - runtime.set_sequences({"Main": mock_seq}) - assert runtime.get_sequence("Main") is mock_seq - - def test_append_result_to_root(self, runtime): - """Verify appending a result with no parent adds to the root results list.""" - result = StepResult() - runtime.append_result(None, result) - assert result in runtime.get_results() +def runtime(): + Runtime.stop_event.clear() + return Runtime(queue.SimpleQueue(), queue.SimpleQueue()) - def test_append_result_to_parent(self, runtime): - """Verify appending a result with a parent UUID adds it as a subresult.""" - parent = StepResult() - runtime.append_result(None, parent) - child = StepResult() - runtime.append_result(parent.uuid, child) - assert child in parent.subresults - def test_append_subresult_with_explicit_uuid(self, runtime): - """Verify appending a subresult using an explicit parent UUID.""" - parent_step_id = uuid.uuid4() - parent = StepResult() - parent.uuid = parent_step_id +def test_recipe_from_definition_constructs_typed_runtime_state_without_reparse(): + model = definition() + recipe = Recipe.from_definition(model, "fixture.yml") + assert recipe.definition is model + assert recipe.recipe_file_name == "fixture.yml" + assert list(recipe.sequences) == ["Main", "Sub"] + assert isinstance(recipe.sequences["Main"].steps[1], IndexedStep) - runtime.append_result(None, parent) - assert len(runtime.get_results()) == 1 - sub = StepResult() - sub.set_result(ResultType.DONE) - runtime.append_result(parent_step_id, sub) +def test_recipe_path_requires_v2_and_raises_structured_error(tmp_path): + v2 = tmp_path / "v2.yml" + v2.write_text(dump_recipe(definition()), encoding="utf-8") + assert Recipe(v2).main_sequence == "Main" - assert len(parent.subresults) == 1 - assert parent.subresults[0] == sub + legacy = tmp_path / "v1.yml" + legacy.write_text(v2.read_text(encoding="utf-8").replace("2.0.0", "1.0.0")) + with pytest.raises(RecipeParseError) as caught: + Recipe(legacy) + assert "unsupported-recipe-version" in {d.code for d in caught.value.diagnostics} - def test_send_event(self, runtime): - """Verify that send_event places the event tuple in the event queue.""" - runtime.send_event("test_event", "data1", "data2") - event = runtime.event_queue.get() - assert event == ("test_event", ("data1", "data2")) - def test_send_event_with_mock_queue(self, runtime): - """Verify send_event calls put() on the event queue with correct arguments.""" - mock_event_queue = MagicMock() - runtime.event_queue = mock_event_queue - runtime.send_event('event_test', 'data1', 2) - mock_event_queue.put.assert_called_once_with(('event_test', ('data1', 2))) +def test_registry_exactly_matches_authorable_discriminators(): + discriminators = { + model.model_fields["steptype"].examples[0] + for model in STEP_MODELS + } + assert set(STEP_TYPE_REGISTRY) == discriminators + + +@pytest.mark.parametrize("name", STEP_EXAMPLES) +def test_every_typed_definition_builds_its_concrete_executable(name): + typed = TypeAdapter(AuthorableStep).validate_python(STEP_EXAMPLES[name]) + executable = Step.build_step(typed) + assert type(executable).__name__ == name + assert executable.input_mapping == typed.model_dump( + mode="python", by_alias=True + )["input_mapping"] + + +def test_indexed_direct_input_is_the_only_source_of_runtime_wrapping(): + typed = definition().sequences[0].steps[0] + executable = Step.build_step(typed) + assert isinstance(executable, IndexedStep) + assert executable.template_step.input_mapping["wait_time"]["indexed"] is True + + +def test_setup_main_nested_and_teardown_execute_with_existing_runtime_behavior(): + recipe = Recipe.from_definition(definition()) + active_runtime = runtime() + active_runtime.set_globals(recipe.globals) + active_runtime.set_sequences(recipe.sequences) + result = recipe.sequences["Main"].run(active_runtime, {}) + assert result == ResultType.DONE + assert [item.step.name for item in active_runtime.results] == [ + "setup", "indexed", "nested", "teardown" + ] + + +def test_recipe_run_constructs_top_level_sequence_step_directly(monkeypatch): + recipe = Recipe.from_definition(definition()) + active_runtime = runtime() + monkeypatch.setattr(pypts.recipe.time, "sleep", lambda _: None) + monkeypatch.setattr( + Step, + "build_step", + staticmethod(lambda _: (_ for _ in ()).throw(AssertionError("synthetic dict factory used"))), + ) + results = recipe.run(active_runtime) + assert results + assert active_runtime.recipe_name == recipe.name diff --git a/tests/unit_tests/test_recipe_language.py b/tests/unit_tests/test_recipe_language.py index 1848219..f65e3d5 100644 --- a/tests/unit_tests/test_recipe_language.py +++ b/tests/unit_tests/test_recipe_language.py @@ -1,111 +1,407 @@ # SPDX-FileCopyrightText: 2026 CERN -# # SPDX-License-Identifier: LGPL-2.1-or-later +"""Production recipe-language model and parser evaluation suite.""" +from __future__ import annotations + +import ast +import json from pathlib import Path import pytest import yaml +from pydantic import ValidationError + +from pypts.recipe_artifacts import ( + DEFAULT_REFERENCE_PATH, + DEFAULT_SCHEMA_PATH, + render_json_schema, +) +from pypts.recipe_language import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS +from pypts.recipe_parser import ( + RecipeParseError, + dump_recipe, + parse_recipe_file, + parse_recipe_text, +) +from pypts.recipe_reference import render_reference -from pypts.recipe_language import STEP_SPECS, canonical_step_type, validate_recipe_documents +ROOT = Path(__file__).parents[2] +RECIPES = ROOT / "src" / "pypts" / "recipes" -RECIPES = Path(__file__).parents[2] / "src" / "pypts" / "recipes" +def header(**updates): + value = { + "name": "Pydantic spike", + "version": "1.0", + "recipe_version": "2.0.0", + "description": "Candidate recipe.", + "main_sequence": "Main", + "globals": {}, + } + value.update(updates) + return value -def _recipe_with_step(step, *, contextual=True): - sequence = { +def sequence(steps=None, **updates): + value = { "sequence_name": "Main", "description": "Main sequence.", - "parameters": {}, "outputs": {}, "locals": {}, - "setup_steps": [], "steps": [step], "teardown_steps": [], - } - step_type = canonical_step_type(step["steptype"]) - documents = [ - { - "name": "Language contract", - "version": "1.0", - "recipe_version": "1.0.0", - "description": "Contract fixture.", - "main_sequence": "Main", - "globals": {"ssh_client": None, "host": "target", "user": "root", "port": 22, "password": "secret"}, - }, - ] - if contextual and step_type == "SequenceStep": - documents.append({ - "sequence_name": step["sequence"]["name"], "description": "Target sequence.", - "parameters": {}, "outputs": {}, "locals": {}, - "setup_steps": [], "steps": [], "teardown_steps": [], - }) - if contextual and step_type == "SSHUploadStep": - sequence["setup_steps"] = [{ - "steptype": "SSHConnectStep", "step_name": "Connect", "description": "Connect.", - }] - sequence["teardown_steps"] = [{ - "steptype": "SSHCloseStep", "step_name": "Close", "description": "Close.", - }] - documents.append(sequence) - return documents + "parameters": {}, + "outputs": {}, + "locals": {}, + "setup_steps": [], + "steps": steps or [], + "teardown_steps": [], + } + value.update(updates) + return value + +def source(*documents): + return yaml.safe_dump_all(documents, explicit_start=True, sort_keys=False) -@pytest.mark.parametrize("path", sorted(path for path in RECIPES.glob("*.yml") if path.name != "subsequence_executions_draft.yml")) -def test_working_recipe_corpus_conforms(path): - result = validate_recipe_documents(yaml.safe_load_all(path.read_text(encoding="utf-8"))) - assert result.is_valid, result.errors + +def common(kind, **updates): + value = { + "steptype": kind, + "step_name": kind, + "description": f"Exercise {kind}.", + "input_mapping": {}, + "output_mapping": {}, + } + value.update(updates) + return value + + +STEP_EXAMPLES = { + "PythonModuleStep": common( + "PythonModuleStep", action_type="method", module="tests.py", method_name="run" + ), + "SequenceStep": common( + "SequenceStep", sequence={"type": "internal", "name": "Target"} + ), + "UserInteractionStep": common("UserInteractionStep"), + "WaitStep": common( + "WaitStep", input_mapping={"wait_time": {"type": "direct", "value": 0}} + ), + "UserLoadingStep": common( + "UserLoadingStep", + file_save_location={"type": "local", "variable": "selected"}, + ), + "UserRunMethodStep": common( + "UserRunMethodStep", + trigger_response="run", + action_type="method", + module="tests.py", + method_name="run", + ), + "UserWriteStep": common("UserWriteStep"), + "SerialNumberStep": common("SerialNumberStep"), + "SSHConnectStep": common("SSHConnectStep"), + "SSHCloseStep": common("SSHCloseStep"), + "SSHUploadStep": common( + "SSHUploadStep", + files=[{"local": "bin/tool", "remote": "/tmp/tool"}], + permissions="0755", + skip_if_sha256_match=True, + local_package="fixtures", + ), +} + + +def recipe_for_step(step): + globals_value = { + "ssh_client": None, + "host": "target", + "user": "root", + "port": 22, + "password": "secret", + } + main = sequence([step]) + documents = [header(globals=globals_value), main] + if step["steptype"] == "SequenceStep": + documents.append(sequence(sequence_name="Target")) + elif step["steptype"] == "SSHUploadStep": + main["setup_steps"] = [common("SSHConnectStep")] + main["teardown_steps"] = [common("SSHCloseStep")] + elif step["steptype"] == "SSHConnectStep": + main["teardown_steps"] = [common("SSHCloseStep")] + return source(*documents) -def test_empty_draft_is_not_a_recipe(): - draft = RECIPES / "subsequence_executions_draft.yml" - result = validate_recipe_documents(yaml.safe_load_all(draft.read_text(encoding="utf-8"))) - assert [item.code for item in result.errors] == ["empty-recipe"] +@pytest.mark.parametrize("model", STEP_MODELS, ids=lambda model: model.__name__) +def test_every_step_validates_serializes_and_reparses(model): + first = parse_recipe_text(recipe_for_step(STEP_EXAMPLES[model.__name__])) + assert first.is_valid, first.errors + second = parse_recipe_text(dump_recipe(first.require_recipe())) + assert second.is_valid, second.errors + assert second.recipe == first.recipe -@pytest.mark.parametrize("spec", [spec for spec in STEP_SPECS if spec.source_allowed], ids=lambda spec: spec.name) -def test_every_step_spec_has_a_valid_canonical_example(spec): - result = validate_recipe_documents(_recipe_with_step(dict(spec.example))) - assert result.is_valid, result.errors +INPUT_EXAMPLES = { + "DirectInput": {"type": "direct", "value": [1, 2], "indexed": True}, + "LocalInput": {"type": "local", "local_name": "local_value"}, + "GlobalInput": {"type": "global", "global_name": "global_value"}, + "MethodInput": {"type": "method", "value": "helper"}, +} -def test_step_types_are_case_insensitive_but_have_one_canonical_name(): - assert canonical_step_type("pythonmodulestep") == "PythonModuleStep" - assert canonical_step_type("UnknownStep") is None +@pytest.mark.parametrize("model", INPUT_MODELS, ids=lambda model: model.__name__) +def test_every_input_mapping_validates_serializes_and_reparses(model): + step = STEP_EXAMPLES["PythonModuleStep"] | { + "input_mapping": {"example": INPUT_EXAMPLES[model.__name__]} + } + first = parse_recipe_text(recipe_for_step(step)) + assert isinstance(first.require_recipe().sequences[0].steps[0].input_mapping["example"], model) + assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + + +OUTPUT_EXAMPLES = { + "PassFailOutput": {"type": "passfail"}, + "EqualsOutput": {"type": "equals", "value": 3}, + "RangeOutput": {"type": "range", "min": 1, "max": 4}, + "PassthroughOutput": {"type": "passthrough"}, + "LocalOutput": {"type": "local", "local_name": "saved"}, + "GlobalOutput": {"type": "global", "global_name": "saved"}, + "ImageOutput": {"type": "image"}, +} + + +@pytest.mark.parametrize("model", OUTPUT_MODELS, ids=lambda model: model.__name__) +def test_every_output_mapping_validates_serializes_and_reparses(model): + step = STEP_EXAMPLES["PythonModuleStep"] | { + "output_mapping": {"example": OUTPUT_EXAMPLES[model.__name__]} + } + first = parse_recipe_text(recipe_for_step(step)) + assert isinstance(first.require_recipe().sequences[0].steps[0].output_mapping["example"], model) + assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + + +def test_defaults_are_typed_dumped_and_models_are_frozen(): + recipe = parse_recipe_text(recipe_for_step(STEP_EXAMPLES["UserInteractionStep"])).require_recipe() + step = recipe.sequences[0].steps[0] + assert step.skip is step.critical is step.continue_on_error is False + assert recipe.header.report == "overwrite" + dumped = dump_recipe(recipe) + assert "report: overwrite" in dumped and "skip: false" in dumped + with pytest.raises(ValidationError): + step.skip = True + + +def test_strict_types_unknown_fields_and_structural_rules_are_rejected(): + bad = STEP_EXAMPLES["PythonModuleStep"] | { + "skip": 0, + "surprise": True, + "method_name": None, + } + codes = {item.code for item in parse_recipe_text(recipe_for_step(bad)).errors} + assert {"invalid-field-type", "unknown-field"} <= codes + + missing_method = STEP_EXAMPLES["PythonModuleStep"] | {"method_name": None} + assert "missing-method-name" in { + item.code for item in parse_recipe_text(recipe_for_step(missing_method)).errors + } + wait = common("WaitStep") + assert "missing-required-input" in { + item.code for item in parse_recipe_text(recipe_for_step(wait)).errors + } -def test_internal_indexed_step_cannot_be_written_in_a_recipe(): - step = { - "steptype": "IndexedStep", "step_name": "Internal", "description": "Internal wrapper.", - "input_mapping": {}, "output_mapping": {}, + nested = STEP_EXAMPLES["PythonModuleStep"] | { + "input_mapping": {"local": {"type": "local", "local_name": 1}} } - codes = {item.code for item in validate_recipe_documents(_recipe_with_step(step)).errors} - assert "internal-step-type" in codes + finding = next( + item for item in parse_recipe_text(recipe_for_step(nested)).errors + if item.code == "invalid-field-type" + ) + assert finding.path[-2:] == ("local", "local_name") -def test_implicit_direct_input_is_accepted_as_legacy_syntax(): - step = { - "steptype": "WaitStep", "step_name": "Wait", "description": "Wait.", - "input_mapping": {"wait_time": {"value": 1}}, "output_mapping": {}, +def test_discriminators_are_explicit_and_canonical(): + lowercase = common( + "waitstep", input_mapping={"wait_time": {"type": "direct", "value": 1}} + ) + omitted_type = STEP_EXAMPLES["WaitStep"] | { + "input_mapping": {"wait_time": {"value": 1}} + } + unknown = common("InventedStep") + missing_output_type = STEP_EXAMPLES["PythonModuleStep"] | { + "output_mapping": {"result": {"value": 1}} + } + assert {item.code for item in parse_recipe_text(recipe_for_step(lowercase)).errors} == { + "noncanonical-step-type" + } + assert {item.code for item in parse_recipe_text(recipe_for_step(omitted_type)).errors} == { + "missing-input-type" } - assert validate_recipe_documents(_recipe_with_step(step)).is_valid + assert {item.code for item in parse_recipe_text(recipe_for_step(unknown)).errors} == { + "unknown-step-type" + } + assert { + item.code for item in parse_recipe_text(recipe_for_step(missing_output_type)).errors + } == {"missing-output-type"} + + +def test_v1_migration_errors_are_aggregated_across_documents(): + legacy = header(recipe_version="1.0.0") + first = sequence( + [common("waitstep", input_mapping={"wait_time": {"value": 1}})], + serial_number=12, + ) + second = sequence( + [STEP_EXAMPLES["WaitStep"] | { + "input_mapping": {"wait_time": {"value": 1}} + }], + sequence_name="Other", + serial_number="old", + ) + result = parse_recipe_text(source(legacy, first, second), "legacy.yml") + codes = [item.code for item in result.errors] + assert codes.count("removed-sequence-field") == 2 + assert {"unsupported-recipe-version", "noncanonical-step-type", "missing-input-type"} <= set(codes) + assert all(item.source_name == "legacy.yml" and item.span is not None for item in result.errors) + + +def test_source_spans_point_to_fields_and_nearest_parent(): + text = source(header(main_sequence="Missing"), sequence()) + result = parse_recipe_text(text, "broken.yml") + finding = next(item for item in result.errors if item.code == "unknown-main-sequence") + expected = next( + index for index, line in enumerate(text.splitlines(), start=1) + if line.startswith("main_sequence:") + ) + assert finding.source_name == "broken.yml" + assert finding.span is not None and finding.span.start.line == expected + + missing = source(header(), sequence()).replace("description: Main sequence.\n", "") + finding = next( + item for item in parse_recipe_text(missing).errors + if item.code == "missing-field" and item.path[-1] == "description" + ) + assert finding.span is not None and finding.span.start.line > 1 + + +def test_yaml_failures_duplicate_keys_and_recursive_aliases(): + malformed = parse_recipe_text("name: [unterminated") + unsafe = parse_recipe_text("!!python/object:builtins.object {}") + duplicate = parse_recipe_text( + source(header(), sequence()).replace("name: Pydantic spike", "name: First\nname: Second") + ) + recursive = parse_recipe_text("---\n&a {name: *a}\n") + assert {item.code for item in malformed.errors} == {"yaml-syntax-error"} + assert "unsafe-yaml" in {item.code for item in unsafe.errors} + assert "duplicate-key" in {item.code for item in duplicate.errors} + assert "recursive-alias" in {item.code for item in recursive.errors} + + +def test_file_api_empty_sources_and_require_recipe(tmp_path): + path = tmp_path / "recipe.yml" + path.write_text(source(header(), sequence()), encoding="utf-8") + assert parse_recipe_file(path).is_valid + assert parse_recipe_file(tmp_path / "missing.yml").errors[0].code == "file-read-error" + assert parse_recipe_text(None).errors[0].code == "invalid-source" + assert parse_recipe_text(" # comment only\n").errors[0].code == "empty-recipe" + result = parse_recipe_text("") + with pytest.raises(RecipeParseError) as error: + result.require_recipe() + assert error.value.diagnostics == result.diagnostics -def test_mapping_and_sequence_contract_failures_are_reported(): - step = { - "steptype": "SequenceStep", "step_name": "Missing", "description": "Missing target.", - "sequence": {"type": "internal", "name": "NoSuchSequence"}, - "input_mapping": { +def test_cross_document_semantic_rules_report_custom_codes(): + nested = common( + "SequenceStep", + sequence={"type": "internal", "name": "Missing"}, + input_mapping={ "left": {"type": "direct", "value": [1], "indexed": True}, "right": {"type": "direct", "value": [1, 2], "indexed": True}, }, - "output_mapping": {"result": {"type": "passthrough"}, "passed": {"type": "passfail"}}, - } - codes = {item.code for item in validate_recipe_documents(_recipe_with_step(step, contextual=False)).errors} - assert {"unequal-indexed-inputs", "mixed-passthrough", "unknown-sequence-reference"} <= codes + output_mapping={ + "result": {"type": "passthrough"}, + "passed": {"type": "passfail"}, + }, + ) + duplicate = sequence(sequence_name="Main") + result = parse_recipe_text(source(header(), sequence([nested]), duplicate)) + assert { + "duplicate-sequence", + "unknown-sequence-reference", + "unequal-indexed-inputs", + "mixed-passthrough", + } <= {item.code for item in result.errors} -def test_ssh_lifecycle_is_part_of_the_contract(): - step = { - "steptype": "SSHUploadStep", "step_name": "Upload", "description": "Upload.", - "files": [], "output_mapping": {}, +def test_ssh_context_and_ordering_are_semantic_rules(): + upload = STEP_EXAMPLES["SSHUploadStep"] + unclosed = sequence([upload], setup_steps=[common("SSHConnectStep")]) + result = parse_recipe_text(source(header(), unclosed)) + codes = {item.code for item in result.errors} + assert {"missing-ssh-global", "missing-ssh-credential", "missing-ssh-close"} <= codes + + before_connect = sequence([upload, common("SSHConnectStep")], teardown_steps=[common("SSHCloseStep")]) + assert "missing-ssh-connect" in { + item.code for item in parse_recipe_text(source(header(), before_connect)).errors } - result = validate_recipe_documents(_recipe_with_step(step, contextual=False)) - assert "missing-ssh-connect" in {item.code for item in result.errors} + + +@pytest.mark.parametrize( + "path", + sorted( + path for path in RECIPES.glob("*.yml") + if path.name != "subsequence_executions_draft.yml" + ), +) +def test_bundled_corpus_is_rejected_until_phase_7(path): + result = parse_recipe_file(path) + assert not result.is_valid + assert "unsupported-recipe-version" in {item.code for item in result.errors} + + +def test_raw_legacy_corpus_exposes_migration_diagnostics(): + results = [ + parse_recipe_file(path) + for path in RECIPES.glob("*.yml") + if path.name != "subsequence_executions_draft.yml" + ] + assert all("unsupported-recipe-version" in {item.code for item in result.errors} for result in results) + all_codes = {item.code for result in results for item in result.errors} + assert {"noncanonical-step-type", "missing-input-type", "removed-sequence-field"} <= all_codes + + +def test_generated_schema_and_reference_are_complete_and_current(): + schema_text = render_json_schema() + schema = json.loads(schema_text) + reference = render_reference(schema) + assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") + assert reference == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + definitions = schema["$defs"] + for model in STEP_MODELS + INPUT_MODELS + OUTPUT_MODELS: + assert model.__name__ in definitions + kind = model.model_fields.get("steptype") or model.model_fields["type"] + anchor_kind = kind.examples[0].lower() + group = "step" if model in STEP_MODELS else "input" if model in INPUT_MODELS else "output" + assert reference.count(f".. _recipe-v2-{group}-{anchor_kind}:") == 1 + + report = STEP_MODELS[0].model_fields["skip"] + assert report.description in reference + assert 'default ``false``' in reference + assert 'Example: ``false``.' in reference + + +def test_spike_has_no_runtime_gui_yamview_or_sphinx_imports(): + imported = set() + for path in Path(__file__).parent.glob("*.py"): + if path.name.startswith("test_"): + continue + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} + assert not any( + name == item or name.startswith(item + ".") + for name in imported + for item in forbidden + ) diff --git a/tests/unit_tests/test_recipe_parser.py b/tests/unit_tests/test_recipe_parser.py index b971e47..94f18f5 100644 --- a/tests/unit_tests/test_recipe_parser.py +++ b/tests/unit_tests/test_recipe_parser.py @@ -1,207 +1,23 @@ # SPDX-FileCopyrightText: 2026 CERN -# # SPDX-License-Identifier: LGPL-2.1-or-later +"""Public production parser API checks.""" -import ast -from pathlib import Path -import textwrap +from pypts.recipe_parser import RecipeParseError, parse_recipe_file, parse_recipe_text -import pytest -from pypts.recipe_parser import ( - DirectInput, - EqualsOutput, - GlobalInput, - GlobalOutput, - ImageOutput, - LocalInput, - LocalOutput, - MethodInput, - PassFailOutput, - PassthroughOutput, - RangeOutput, - RecipeParseError, - dump_recipe, - parse_recipe_file, - parse_recipe_text, -) - - -RECIPES = Path(__file__).parents[2] / "src" / "pypts" / "recipes" -PARSER = Path(__file__).parents[2] / "src" / "pypts" / "recipe_parser.py" - - -def recipe_text(steps="[]", *, recipe_version="1.0.0"): - raw_steps = textwrap.dedent(steps).strip() - steps_value = f" {raw_steps}" if raw_steps == "[]" else "\n" + textwrap.indent(raw_steps, " ") - return f"""--- -name: Parser test -version: "1" -recipe_version: {recipe_version} -description: Parser test recipe -main_sequence: Main -globals: {{}} ---- -sequence_name: Main -description: Main sequence -parameters: {{}} -outputs: {{}} -locals: {{}} -setup_steps: [] -steps:{steps_value} -teardown_steps: [] -""" - - -def test_parse_builds_typed_mappings_and_defaults(): - steps = """ - - steptype: PythonModuleStep - step_name: typed mappings - description: Exercise all mapping models - action_type: method - module: tests.py - method_name: run - input_mapping: - direct: {type: direct, value: [1, 2], indexed: true} - local: {type: local, local_name: local_value} - global: {type: global, global_name: global_value} - method: {type: method, value: helper} - output_mapping: - passed: {type: passfail} - exact: {type: equals, value: 3} - bounded: {type: range, min: 1, max: 4} - local: {type: local, local_name: saved} - global: {type: global, global_name: saved} - chart: {type: image} - - steptype: SequenceStep - step_name: passthrough - description: Exercise passthrough - sequence: {type: internal, name: Main} - output_mapping: - result: {type: passthrough} - """ - result = parse_recipe_text(recipe_text(steps)) - recipe = result.require_recipe() - first, second = recipe.sequences[0].steps - - assert first.skip is first.critical is first.continue_on_error is False - assert isinstance(first.input_mapping["direct"], DirectInput) - assert isinstance(first.input_mapping["local"], LocalInput) - assert isinstance(first.input_mapping["global"], GlobalInput) - assert isinstance(first.input_mapping["method"], MethodInput) - assert isinstance(first.output_mapping["passed"], PassFailOutput) - assert isinstance(first.output_mapping["exact"], EqualsOutput) - assert isinstance(first.output_mapping["bounded"], RangeOutput) - assert isinstance(first.output_mapping["local"], LocalOutput) - assert isinstance(first.output_mapping["global"], GlobalOutput) - assert isinstance(first.output_mapping["chart"], ImageOutput) - assert isinstance(second.output_mapping["result"], PassthroughOutput) - - -def test_normalization_warnings_and_canonical_dump(): - steps = """ - - steptype: waitstep - step_name: wait - description: Legacy spelling - input_mapping: - wait_time: {value: 1} - """ - result = parse_recipe_text(recipe_text(steps), "legacy.yml") - assert {warning.code for warning in result.warnings} == { - "noncanonical-step-type", "implicit-direct-input", - } - canonical = dump_recipe(result.require_recipe()) - assert "steptype: WaitStep" in canonical - assert "type: direct" in canonical - assert "skip: false" in canonical - assert not parse_recipe_text(canonical).warnings - - -def test_contract_diagnostic_has_source_and_nearest_span(): - text = recipe_text("[]").replace("main_sequence: Main", "main_sequence: Missing") - result = parse_recipe_text(text, "broken.yml") - diagnostic = next(item for item in result.errors if item.code == "unknown-main-sequence") - assert diagnostic.source_name == "broken.yml" - assert diagnostic.span is not None - expected_line = next(index for index, line in enumerate(text.splitlines(), start=1) if line.startswith("main_sequence:")) - assert diagnostic.span.start.line == expected_line - assert diagnostic.span.start.column > 1 - - -def test_missing_field_uses_parent_span(): - text = recipe_text("[]").replace("description: Main sequence\n", "") - result = parse_recipe_text(text, "missing.yml") - diagnostic = next(item for item in result.errors if item.code == "missing-field") - assert diagnostic.path[-1] == "description" - assert diagnostic.span is not None - assert diagnostic.span.start.line > 1 - - -def test_duplicate_yaml_keys_are_rejected(): - text = recipe_text("[]").replace("name: Parser test", "name: First\nname: Second") - result = parse_recipe_text(text) - assert "duplicate-key" in {item.code for item in result.errors} - assert result.recipe is None - - -def test_malformed_and_unsafe_yaml_are_rejected(): - malformed = parse_recipe_text("name: [unterminated") - unsafe = parse_recipe_text("!!python/object:builtins.object {}") - assert {item.code for item in malformed.errors} == {"yaml-syntax-error"} - assert "unsafe-yaml" in {item.code for item in unsafe.errors} - - -def test_empty_and_non_text_sources_are_rejected(): - assert {item.code for item in parse_recipe_text(" \n").errors} == {"empty-recipe"} - assert {item.code for item in parse_recipe_text(None).errors} == {"invalid-source"} - - -def test_file_entry_point_and_read_failure(tmp_path): - path = tmp_path / "recipe.yml" - path.write_text(recipe_text("[]"), encoding="utf-8") - from_file = parse_recipe_file(path) - from_text = parse_recipe_text(path.read_text(encoding="utf-8"), str(path)) - assert from_file.recipe == from_text.recipe - assert parse_recipe_file(tmp_path / "missing.yml").errors[0].code == "file-read-error" - - -def test_require_recipe_raises_with_diagnostics(): - result = parse_recipe_text("") - with pytest.raises(RecipeParseError) as error: +def test_invalid_file_result_raises_structured_parse_error(tmp_path): + path = tmp_path / "legacy.yml" + path.write_text("name: legacy\nrecipe_version: 1.0.0\n", encoding="utf-8") + result = parse_recipe_file(path) + assert result.errors + try: result.require_recipe() - assert error.value.diagnostics == result.diagnostics - - -def test_unsupported_recipe_version_is_rejected(): - result = parse_recipe_text(recipe_text("[]", recipe_version="2.0.0")) - assert "unsupported-recipe-version" in {item.code for item in result.errors} - - -@pytest.mark.parametrize( - "path", - sorted(path for path in RECIPES.glob("*.yml") if path.name != "subsequence_executions_draft.yml"), -) -def test_bundled_recipe_parse_dump_reparse_is_model_stable(path): - first = parse_recipe_file(path) - assert first.is_valid, first.errors - canonical = dump_recipe(first.require_recipe()) - second = parse_recipe_text(canonical, f"canonical:{path.name}") - assert second.is_valid, second.errors - assert second.recipe == first.recipe - - -def test_comment_only_draft_is_rejected_as_empty(): - result = parse_recipe_file(RECIPES / "subsequence_executions_draft.yml") - assert {item.code for item in result.errors} == {"empty-recipe"} + except RecipeParseError as error: + assert error.diagnostics == result.diagnostics + else: + raise AssertionError("require_recipe() accepted an invalid recipe") -def test_parser_has_no_runtime_gui_or_docs_imports(): - tree = ast.parse(PARSER.read_text(encoding="utf-8")) - imported = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imported.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - imported.add(node.module) - forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} - assert not any(name == item or name.startswith(item + ".") for name in imported for item in forbidden) +def test_non_text_source_is_a_structured_error(): + result = parse_recipe_text(None) # type: ignore[arg-type] + assert [item.code for item in result.errors] == ["invalid-source"] diff --git a/tests/unit_tests/test_recipe_pydantic_docs.py b/tests/unit_tests/test_recipe_pydantic_docs.py index 105472e..1c477fd 100644 --- a/tests/unit_tests/test_recipe_pydantic_docs.py +++ b/tests/unit_tests/test_recipe_pydantic_docs.py @@ -8,23 +8,23 @@ import json from pathlib import Path -from spikes.recipe_pydantic.artifacts import ( +from pypts.recipe_artifacts import ( main, render_json_schema, rendered_artifacts, write_artifacts, ) -from spikes.recipe_pydantic.parser import ( +from pypts.recipe_parser import ( dump_recipe, parse_recipe_file, parse_recipe_text, ) -from spikes.recipe_pydantic.reference import render_reference +from pypts.recipe_reference import render_reference ROOT = Path(__file__).parents[2] DOC_RECIPE = ROOT / "docs" / "source" / "_examples" / "recipe_v2.yml" ARCHITECTURE = ROOT / "docs" / "source" / "recipe_language_architecture.rst" -REFERENCE_RENDERER = ROOT / "spikes" / "recipe_pydantic" / "reference.py" +REFERENCE_RENDERER = ROOT / "src" / "pypts" / "recipe_reference.py" def _mapping(schema, name): @@ -115,10 +115,10 @@ def test_documentation_recipe_is_warning_free_and_model_stable(): def test_literalinclude_markers_are_unique_and_paired(): architecture = ARCHITECTURE.read_text(encoding="utf-8") sources = { - "models.py": (ROOT / "spikes" / "recipe_pydantic" / "models.py").read_text( + "models.py": (ROOT / "src" / "pypts" / "recipe_language.py").read_text( encoding="utf-8" ), - "parser.py": (ROOT / "spikes" / "recipe_pydantic" / "parser.py").read_text( + "parser.py": (ROOT / "src" / "pypts" / "recipe_parser.py").read_text( encoding="utf-8" ), } diff --git a/tests/unit_tests/test_recipe_reference.py b/tests/unit_tests/test_recipe_reference.py index 2c979bd..42efaad 100644 --- a/tests/unit_tests/test_recipe_reference.py +++ b/tests/unit_tests/test_recipe_reference.py @@ -1,261 +1,46 @@ # SPDX-FileCopyrightText: 2026 CERN -# # SPDX-License-Identifier: LGPL-2.1-or-later +"""Production JSON-Schema-to-RST renderer checks.""" import ast +import json from pathlib import Path -import pytest -import yaml - -from pypts.recipe_language import ( - COMMON_STEP_SPEC, - CONSTRAINT_SPECS, - DOCUMENTED_DIAGNOSTIC_CODES, - FILE_SAVE_LOCATION_SPEC, - HEADER_SPEC, - INPUT_MAPPING_SPECS, - OUTPUT_MAPPING_SPECS, - SEQUENCE_REFERENCE_SPEC, - SEQUENCE_SPEC, - STEP_SPECS, -) -from pypts.recipe_parser import ( - DirectInput, - EqualsOutput, - GlobalInput, - GlobalOutput, - ImageOutput, - LocalInput, - LocalOutput, - MethodInput, - PassFailOutput, - PassthroughOutput, - RangeOutput, - parse_recipe_text, -) -from pypts.recipe_reference import ( - _fixture_text, - check_recipe_reference, - main, - render_recipe_reference, -) - +from pypts.recipe_artifacts import render_json_schema +from pypts.recipe_reference import render_reference ROOT = Path(__file__).parents[2] -REFERENCE = ROOT / "docs" / "generated" / "recipe_language_reference.rst" - - -def _all_structures(): - return ( - HEADER_SPEC, - SEQUENCE_SPEC, - COMMON_STEP_SPEC, - SEQUENCE_REFERENCE_SPEC, - FILE_SAVE_LOCATION_SPEC, - ) - - -def test_registry_metadata_is_complete_and_unambiguous(): - step_names = [spec.name.casefold() for spec in STEP_SPECS] - assert len(step_names) == len(set(step_names)) - - for structure in _all_structures(): - assert structure.name and structure.description - field_names = [field.name for field in structure.fields] - assert len(field_names) == len(set(field_names)) - assert all(field.description for field in structure.fields) - - for specs in (INPUT_MAPPING_SPECS, OUTPUT_MAPPING_SPECS): - names = [spec.name for spec in specs] - assert len(names) == len(set(names)) - for spec in specs: - assert spec.description and spec.example - assert spec.example["type"] == spec.name - assert all(field.description for field in spec.fields) - - for spec in STEP_SPECS: - assert spec.name and spec.description and spec.example - assert all(field.description for field in spec.fields) - -def test_constraint_diagnostic_registry_is_complete_and_unique(): - constraint_codes = [spec.code for spec in CONSTRAINT_SPECS] - diagnostic_codes = [code for spec in CONSTRAINT_SPECS for code in spec.diagnostic_codes] - assert len(constraint_codes) == len(set(constraint_codes)) - assert len(diagnostic_codes) == len(set(diagnostic_codes)) - assert set(diagnostic_codes) == DOCUMENTED_DIAGNOSTIC_CODES - assert all(spec.scope and spec.description and spec.diagnostic_codes for spec in CONSTRAINT_SPECS) - discovered = set() - for relative in ("src/pypts/recipe_language.py", "src/pypts/recipe_parser.py"): - tree = ast.parse((ROOT / relative).read_text(encoding="utf-8")) - for node in ast.walk(tree): - if not isinstance(node, ast.Call) or not node.args: - continue - function = node.func.id if isinstance(node.func, ast.Name) else None - if function not in {"Diagnostic", "_source_diagnostic"}: - continue - if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): - discovered.add(node.args[0].value) - assert discovered <= DOCUMENTED_DIAGNOSTIC_CODES +def test_reference_renders_every_discriminator_from_generated_json(): + schema = json.loads(render_json_schema()) + rendered = render_reference(schema) + for group, definition in ( + ("step", "Step"), ("input", "InputMapping"), ("output", "OutputMapping") + ): + mapping = schema["$defs"][definition]["discriminator"]["mapping"] + for name in mapping: + assert rendered.count(f".. _recipe-v2-{group}-{name.lower()}:") == 1 -def test_reference_generator_has_no_runtime_gui_or_sphinx_imports(): - tree = ast.parse((ROOT / "src/pypts/recipe_reference.py").read_text(encoding="utf-8")) - imported = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imported.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module: - imported.add(node.module) - forbidden = {"pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} +def test_json_only_renderer_has_no_model_runtime_gui_or_sphinx_imports(): + tree = ast.parse( + (ROOT / "src" / "pypts" / "recipe_reference.py").read_text(encoding="utf-8") + ) + imported = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + imported.update( + node.module + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.module + ) + forbidden = {"pydantic", "pypts.recipe_language", "pypts.recipe", "pypts.steps", "pypts.YamVIEW", "sphinx"} assert not any( - name == item or name.startswith(item + ".") + name == prefix or name.startswith(prefix + ".") for name in imported - for item in forbidden + for prefix in forbidden ) - - -@pytest.mark.parametrize( - "spec", - [spec for spec in STEP_SPECS if spec.source_allowed], - ids=lambda spec: spec.name, -) -def test_every_public_step_example_is_an_executable_parser_fixture(spec): - result = parse_recipe_text(_fixture_text(spec.example), f"fixture:{spec.name}") - assert result.is_valid, result.errors - assert not result.warnings - assert result.require_recipe().sequences[0].steps[0].steptype == spec.name - - -INPUT_TYPES = { - "direct": DirectInput, - "local": LocalInput, - "global": GlobalInput, - "method": MethodInput, -} - - -@pytest.mark.parametrize("spec", INPUT_MAPPING_SPECS, ids=lambda spec: spec.name) -def test_every_input_mapping_example_builds_its_typed_model(spec): - step = { - "steptype": "PythonModuleStep", - "step_name": "Input fixture", - "description": "Input mapping fixture.", - "action_type": "method", - "module": "tests.py", - "method_name": "run", - "input_mapping": {"example": dict(spec.example)}, - "output_mapping": {}, - } - result = parse_recipe_text(_fixture_text(step)) - assert result.is_valid, result.errors - value = result.require_recipe().sequences[0].steps[0].input_mapping["example"] - assert isinstance(value, INPUT_TYPES[spec.name]) - - -OUTPUT_TYPES = { - "passfail": PassFailOutput, - "equals": EqualsOutput, - "range": RangeOutput, - "passthrough": PassthroughOutput, - "local": LocalOutput, - "global": GlobalOutput, - "image": ImageOutput, -} - - -@pytest.mark.parametrize("spec", OUTPUT_MAPPING_SPECS, ids=lambda spec: spec.name) -def test_every_output_mapping_example_builds_its_typed_model(spec): - step = { - "steptype": "PythonModuleStep", - "step_name": "Output fixture", - "description": "Output mapping fixture.", - "action_type": "method", - "module": "tests.py", - "method_name": "run", - "input_mapping": {}, - "output_mapping": {"example": dict(spec.example)}, - } - result = parse_recipe_text(_fixture_text(step)) - assert result.is_valid, result.errors - value = result.require_recipe().sequences[0].steps[0].output_mapping["example"] - assert isinstance(value, OUTPUT_TYPES[spec.name]) - - -def test_mapping_field_types_are_enforced_from_the_registry(): - step = { - "steptype": "PythonModuleStep", - "step_name": "Invalid mapping fields", - "description": "Mapping field type fixture.", - "action_type": "method", - "module": "tests.py", - "method_name": "run", - "input_mapping": {"source": {"type": "local", "local_name": 1}}, - "output_mapping": {"destination": {"type": "global", "global_name": 2}}, - } - result = parse_recipe_text(_fixture_text(step)) - assert {item.code for item in result.errors} == { - "invalid-input-field-type", - "invalid-output-field-type", - } - - -@pytest.mark.parametrize("report", ("overwrite", "append")) -def test_every_report_mode_from_the_registry_parses(report): - step = { - "steptype": "WaitStep", - "step_name": "Wait", - "description": "Report mode fixture.", - "input_mapping": {"wait_time": {"type": "direct", "value": 0}}, - "output_mapping": {}, - } - documents = list(yaml.safe_load_all(_fixture_text(step))) - documents[0]["report"] = report - source = yaml.safe_dump_all(documents, explicit_start=True, sort_keys=False) - assert parse_recipe_text(source).is_valid - - -@pytest.mark.parametrize("action_type", ("method", "read_attribute", "write_attribute")) -def test_every_python_action_type_from_the_registry_parses(action_type): - step = { - "steptype": "PythonModuleStep", - "step_name": "Action fixture", - "description": "Action choice fixture.", - "action_type": action_type, - "module": "tests.py", - "input_mapping": {}, - "output_mapping": {}, - } - if action_type == "method": - step["method_name"] = "run" - assert parse_recipe_text(_fixture_text(step)).is_valid - - -def test_generated_reference_is_complete_and_current(): - rendered = render_recipe_reference() - assert rendered == REFERENCE.read_text(encoding="utf-8") - for spec in STEP_SPECS: - anchor = f".. _recipe-step-{spec.name.lower()}:" - assert rendered.count(anchor) == int(spec.source_allowed) - for spec in INPUT_MAPPING_SPECS: - assert rendered.count(f".. _recipe-input-{spec.name}:") == 1 - for spec in OUTPUT_MAPPING_SPECS: - assert rendered.count(f".. _recipe-output-{spec.name}:") == 1 - - -def test_reference_cli_writes_checks_and_detects_stale_files(tmp_path, capsys): - path = tmp_path / "nested" / "reference.rst" - assert main([str(path)]) == 0 - assert check_recipe_reference(path) - assert main(["--check", str(path)]) == 0 - - path.write_text("stale\n", encoding="utf-8") - assert main(["--check", str(path)]) == 1 - assert path.read_text(encoding="utf-8") == "stale\n" - assert "missing or stale" in capsys.readouterr().err - - missing = tmp_path / "missing.rst" - assert main(["--check", str(missing)]) == 1 - assert not missing.exists() diff --git a/tests/unit_tests/test_steps.py b/tests/unit_tests/test_steps.py index 5976091..923df7f 100644 --- a/tests/unit_tests/test_steps.py +++ b/tests/unit_tests/test_steps.py @@ -394,75 +394,38 @@ def test_empty_step_list(self, mock_runtime): class TestBuildStep: def test_build_wait_step(self): - data = {"steptype": "WaitStep", "step_name": "Wait", "input_mapping": {"wait_time": {"type": "direct", "value": 0.01}}, "output_mapping": {}} - step = Step.build_step(data) - assert isinstance(step, WaitStep) - assert step.name == "Wait" + from pypts.recipe_language import WaitStep as WaitDefinition - def test_build_case_insensitive(self): - data = {"steptype": "waitstep", "step_name": "Wait", "input_mapping": {}, "output_mapping": {}} - step = Step.build_step(data) + definition = WaitDefinition( + steptype="WaitStep", + step_name="Wait", + description="Wait.", + input_mapping={"wait_time": {"type": "direct", "value": 0.01}}, + ) + step = Step.build_step(definition) assert isinstance(step, WaitStep) - - @pytest.mark.parametrize(("name", "expected", "extra"), [ - ("pythonMODULEstep", PythonModuleStep, {"action_type": "method", "module": "m.py", "method_name": "run"}), - ("SEQUENCEstep", SequenceStep, {"sequence": {"type": "internal", "name": "Sub"}}), - ("userinteractionSTEP", UserInteractionStep, {}), - ("WAITstep", WaitStep, {}), - ("userloadingstep", UserLoadingStep, {}), - ("userrunmethodstep", UserRunMethodStep, {}), - ("userwritestep", UserWriteStep, {}), - ("serialnumberstep", SerialNumberStep, {}), - ("sshCONNECTstep", SSHConnectStep, {}), - ("sshCLOSEstep", SSHCloseStep, {}), - ("sshUPLOADstep", SSHUploadStep, {"files": []}), - ]) - def test_registry_supports_every_concrete_step_case_insensitively(self, name, expected, extra): - data = {"steptype": name, "step_name": "S", "input_mapping": {}, "output_mapping": {}, **extra} - assert isinstance(Step.build_step(data), expected) - - def test_unknown_step_type_is_descriptive(self): - with pytest.raises(ValueError, match="Unknown step type 'MagicStep'"): - Step.build_step({"steptype": "MagicStep", "step_name": "S"}) - - def test_build_does_not_mutate_definition(self): - data = {"steptype": "WaitStep", "step_name": "S"} - Step.build_step(data) - assert data["steptype"] == "WaitStep" - - def test_build_sequence_step(self): - data = {"steptype": "SequenceStep", "step_name": "SubSeq", "sequence": {"type": "internal", "name": "Sub"}, "input_mapping": {}, "output_mapping": {}} - step = Step.build_step(data) - assert isinstance(step, SequenceStep) + assert step.name == "Wait" def test_build_indexed_step_wraps_when_indexed(self): - data = { - "steptype": "WaitStep", - "step_name": "IndexedWait", - "input_mapping": {"wait_time": {"type": "direct", "value": [0.01, 0.02], "indexed": True}}, - "output_mapping": {}, - } - step = Step.build_step(data) + from pypts.recipe_language import WaitStep as WaitDefinition + + definition = WaitDefinition( + steptype="WaitStep", + step_name="IndexedWait", + description="Wait twice.", + input_mapping={ + "wait_time": { + "type": "direct", "value": [0.01, 0.02], "indexed": True + } + }, + ) + step = Step.build_step(definition) assert isinstance(step, IndexedStep) assert isinstance(step.template_step, WaitStep) - def test_build_python_module_step(self): - data = { - "steptype": "PythonModuleStep", - "step_name": "Run", - "action_type": "method", - "module": "my_module.py", - "method_name": "my_func", - "input_mapping": {}, - "output_mapping": {}, - } - step = Step.build_step(data) - assert isinstance(step, PythonModuleStep) - - def test_build_serial_number_step(self): - data = {"steptype": "SerialNumberStep", "step_name": "SN", "input_mapping": {}, "output_mapping": {}} - step = Step.build_step(data) - assert isinstance(step, SerialNumberStep) + def test_rejects_unvalidated_dictionary(self): + with pytest.raises(TypeError, match="validated authorable"): + Step.build_step({"steptype": "WaitStep"}) # ============================================================ diff --git a/tests/unit_tests/test_verify_recipe.py b/tests/unit_tests/test_verify_recipe.py index 5eaf1e6..5b4a676 100644 --- a/tests/unit_tests/test_verify_recipe.py +++ b/tests/unit_tests/test_verify_recipe.py @@ -1,178 +1,71 @@ -import textwrap +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later import pytest +from pypts.recipe_parser import dump_recipe, parse_recipe_file from pypts.YamVIEW.verify_recipe import ( RecipeValidationError, validate_recipe_file, + validate_recipe_filepath, validate_recipe_string_variable, ) - -def write_recipe(tmp_path, body): +VALID = """--- +name: Valid +version: '1' +recipe_version: 2.0.0 +description: valid +main_sequence: Main +globals: {} +--- +sequence_name: Main +description: main +parameters: {} +outputs: {} +locals: {} +setup_steps: [] +steps: +- steptype: WaitStep + step_name: wait + description: wait + input_mapping: + wait_time: {type: direct, value: 0} +teardown_steps: [] +""" + + +def test_file_and_filepath_wrappers_accept_v2(tmp_path): path = tmp_path / "recipe.yml" - path.write_text(textwrap.dedent(body)) - return path - - -def test_validates_setup_main_and_teardown_semantics(tmp_path): - path = write_recipe(tmp_path, """ - name: Valid - version: "1" - description: valid - globals: {} - --- - sequence_name: Main - description: main - parameters: {} - outputs: {} - locals: {} - setup_steps: [] - steps: - - steptype: WaitStep - step_name: wait - description: wait - input_mapping: {} - output_mapping: {} - teardown_steps: [] - """) - validate_recipe_file(path) + path.write_text(VALID, encoding="utf-8") + assert validate_recipe_file(path) is None + assert validate_recipe_filepath(path) -@pytest.mark.parametrize("section", ["setup_steps", "steps", "teardown_steps"]) -def test_rejects_invalid_step_in_every_section(tmp_path, section): - path = write_recipe(tmp_path, f""" - name: Invalid - version: "1" - description: invalid - globals: {{}} - --- - sequence_name: Main - description: main - parameters: {{}} - outputs: {{}} - locals: {{}} - setup_steps: {"[{steptype: Unknown, step_name: bad}]" if section == "setup_steps" else "[]"} - steps: {"[{steptype: Unknown, step_name: bad}]" if section == "steps" else "[]"} - teardown_steps: {"[{steptype: Unknown, step_name: bad}]" if section == "teardown_steps" else "[]"} - """) - with pytest.raises(RecipeValidationError, match="Validation failed"): +def test_file_wrapper_formats_structured_diagnostics(tmp_path): + path = tmp_path / "legacy.yml" + path.write_text(VALID.replace("2.0.0", "1.0.0"), encoding="utf-8") + with pytest.raises(RecipeValidationError) as caught: validate_recipe_file(path) + assert caught.value.diagnostics + assert "unsupported-recipe-version" in caught.value.faults[0] + assert ":4:" in caught.value.faults[0] + assert not validate_recipe_filepath(path) -def test_rejects_unknown_sequence_index_lengths_and_mixed_passthrough(tmp_path): - path = write_recipe(tmp_path, """ - name: Invalid - version: "1" - description: invalid - globals: {} - --- - sequence_name: Main - description: main - parameters: {} - outputs: {} - locals: {} - setup_steps: [] - steps: - - steptype: SequenceStep - step_name: sub - description: sub - sequence: {type: internal, name: Missing} - input_mapping: - a: {value: [1], indexed: true} - b: {value: [1, 2], indexed: true} - output_mapping: - result: {type: passthrough} - ok: {type: passfail} - teardown_steps: [] - """) - with pytest.raises(RecipeValidationError) as error: - validate_recipe_file(path) - faults = "\n".join(error.value.faults) - assert "equal lengths" in faults - assert "sole verdict" in faults - assert "unknown sequence 'Missing'" in faults - +def test_string_wrapper_returns_diagnostic_message(): + valid, message = validate_recipe_string_variable( + VALID.replace("WaitStep", "waitstep") + ) + assert not valid + assert "noncanonical-step-type" in message + assert "WaitStep" in message -def test_rejects_invalid_mapping_fields_and_invalid_top_level_policy(tmp_path): - path = write_recipe(tmp_path, """ - name: Invalid - version: "1" - description: invalid - continue_on_error: invalid - globals: {} - --- - sequence_name: Main - description: main - parameters: [] - outputs: [] - locals: {} - setup_steps: [] - steps: - - steptype: WaitStep - step_name: wait - description: wait - input_mapping: {} - output_mapping: - measured: {type: range, min: 0} - teardown_steps: [] - """) - with pytest.raises(RecipeValidationError) as error: - validate_recipe_file(path) - faults = "\n".join(error.value.faults) - assert "Top-level 'continue_on_error' should be a boolean" in faults - assert "'parameters' should be a dictionary" in faults - assert "requires 'max'" in faults - -def test_rejects_invalid_ssh_lifecycle(tmp_path): - path = write_recipe(tmp_path, """ - name: Invalid SSH - version: "1" - description: invalid - globals: {ssh_client: null} - --- - sequence_name: Main - description: main - parameters: {} - outputs: {} - locals: {} - setup_steps: [] - steps: - - steptype: SSHUploadStep - step_name: upload - description: upload - files: [] - input_mapping: {} - output_mapping: {} - teardown_steps: [] - """) - with pytest.raises(RecipeValidationError) as error: - validate_recipe_file(path) - faults = "\n".join(error.value.faults) - assert "SSHUploadStep requires SSHConnectStep" in faults - assert "require global 'host'" in faults - - -def test_string_validator_checks_teardown_and_main_sequence(): - valid, message = validate_recipe_string_variable(textwrap.dedent(""" - name: Invalid - version: "1" - description: invalid - main_sequence: Missing - globals: {} - --- - sequence_name: Main - description: main - parameters: {} - outputs: {} - locals: {} - setup_steps: [] - steps: [] - teardown_steps: - - steptype: UnknownStep - step_name: bad - """)) - assert valid is False - assert "Main sequence 'Missing' does not exist" in message - assert "Unknown step type 'UnknownStep'" in message +def test_wrapper_canonical_output_round_trips(tmp_path): + path = tmp_path / "recipe.yml" + path.write_text(VALID, encoding="utf-8") + definition = parse_recipe_file(path).require_recipe() + canonical = dump_recipe(definition) + valid, _ = validate_recipe_string_variable(canonical) + assert valid diff --git a/tests/unit_tests/test_yamview_schema.py b/tests/unit_tests/test_yamview_schema.py new file mode 100644 index 0000000..3428170 --- /dev/null +++ b/tests/unit_tests/test_yamview_schema.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Schema-driven YamVIEW form tests.""" + +from pypts.recipe_language import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS +from pypts.YamVIEW.recipe_step_setup import ( + DiscriminatedMappingWidget, + Step_setup, + recipe_form_description, +) + + +def discriminator(model): + field_name = "steptype" if "steptype" in model.model_fields else "type" + return model.model_fields[field_name].examples[0] + + +def test_form_selectors_and_metadata_come_from_production_schema(): + description = recipe_form_description() + assert set(description["steps"]) == {discriminator(model) for model in STEP_MODELS} + assert set(description["inputs"]) == {discriminator(model) for model in INPUT_MODELS} + assert set(description["outputs"]) == {discriminator(model) for model in OUTPUT_MODELS} + + fields = description["steps"]["PythonModuleStep"]["fields"] + assert fields["module"]["required"] is True + assert fields["module"]["description"] + assert fields["module"]["examples"] + assert fields["skip"]["default"] is False + assert fields["input_mapping"]["widget"] == "structured" + + +def test_step_dialog_round_trips_discriminated_mapping_rows(qapp, qtbot): + dialog = Step_setup() + qtbot.addWidget(dialog) + node = { + "steptype": "WaitStep", + "step_name": "pause", + "description": "Pause briefly.", + "input_mapping": {"wait_time": {"type": "direct", "value": 0}}, + "output_mapping": {"verdict": {"type": "passfail"}}, + } + + dialog.load_definition(node) + assert isinstance( + dialog.schema_form.field_widgets["input_mapping"], + DiscriminatedMappingWidget, + ) + dialog.accept() + + authored = dialog.result_step["_node"] + assert authored["input_mapping"]["wait_time"]["type"] == "direct" + assert authored["input_mapping"]["wait_time"]["value"] == 0 + assert authored["output_mapping"]["verdict"]["type"] == "passfail" From a5f7f354f341a6811e7ab7077922203366391992 Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 10:28:59 +0200 Subject: [PATCH 09/14] renaming to make differences between steps pydantic models and executable steps --- docs/generated/recipe_language_reference.rst | 918 ------------------ docs/source/index.rst | 1 + docs/source/recipe_language_architecture.rst | 15 +- docs/source/recipe_language_maintenance.rst | 125 +++ src/pypts/YamVIEW/recipe_step_setup.py | 18 +- src/pypts/recipe.py | 26 +- src/pypts/recipe_language.py | 40 +- src/pypts/recipe_reference.py | 7 +- tests/unit_tests/test_recipe.py | 9 +- tests/unit_tests/test_recipe_language.py | 31 +- tests/unit_tests/test_recipe_pydantic_docs.py | 14 +- tests/unit_tests/test_recipe_reference.py | 6 +- tests/unit_tests/test_steps.py | 2 +- tests/unit_tests/test_yamview_schema.py | 10 +- 14 files changed, 240 insertions(+), 982 deletions(-) delete mode 100644 docs/generated/recipe_language_reference.rst create mode 100644 docs/source/recipe_language_maintenance.rst diff --git a/docs/generated/recipe_language_reference.rst b/docs/generated/recipe_language_reference.rst deleted file mode 100644 index 9028cc9..0000000 --- a/docs/generated/recipe_language_reference.rst +++ /dev/null @@ -1,918 +0,0 @@ -.. SPDX-FileCopyrightText: 2026 CERN -.. -.. SPDX-License-Identifier: CC-BY-SA-4.0 -.. -.. This file is generated by pypts.recipe_reference. Do not edit it manually. - -Recipe Language Reference -========================= - -Canonical recipe language version: ``1.0.0``. - -Document grammar ----------------- - -A recipe is safe multi-document YAML. The first document is one recipe -header and every following document is one sequence. At least one sequence -is required, and ``main_sequence`` must name one of them. - -Recipe header -------------- - -The first YAML document; identifies the recipe and its entry sequence. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``name`` - - str - - required - - Human-readable recipe name. - * - ``version`` - - str - - required - - Version of this recipe. - * - ``recipe_version`` - - str - - required; allowed: ``1.0.0`` - - Version of the recipe language contract. - * - ``description`` - - str - - required - - Purpose of the recipe. - * - ``main_sequence`` - - str - - required - - Sequence where execution begins. - * - ``globals`` - - dict - - required - - Recipe-wide variables. - * - ``continue_on_error`` - - bool - - optional; default: ``null`` - - Recipe-wide error policy. - * - ``report`` - - str - - optional; default: ``overwrite``; allowed: ``overwrite``, ``append`` - - Report file mode. - * - ``report_name_include_serial`` - - bool - - optional; default: ``false`` - - Include the serial number in the report name. - * - ``test_package`` - - str - - optional; default: ``null`` - - Package containing recipe test modules. - -Sequence --------- - -Each YAML document after the header defines one named sequence. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``sequence_name`` - - str - - required - - Unique sequence name. - * - ``description`` - - str - - required - - Purpose of the sequence. - * - ``parameters`` - - dict - - required - - Reserved sequence input metadata. - * - ``outputs`` - - dict - - required - - Reserved sequence output metadata. - * - ``locals`` - - dict - - required - - Variables local to the sequence. - * - ``setup_steps`` - - list - - required - - Steps run before the main steps. - * - ``steps`` - - list - - required - - Ordered main steps. - * - ``teardown_steps`` - - list - - required - - Steps run during teardown. - * - ``serial_number`` - - str or int - - optional; legacy - - Runtime-ignored legacy sequence metadata. - -Common step fields ------------------- - -These fields are shared by every authorable step type. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``steptype`` - - str - - required - - Registered step type. - * - ``step_name`` - - str - - required - - Human-readable step name. - * - ``description`` - - str - - required - - Purpose of the step. - * - ``id`` - - str - - optional - - Optional stable step identifier. - * - ``skip`` - - bool - - optional; default: ``false`` - - Skip execution. - * - ``critical`` - - bool - - optional; default: ``false`` - - Stop on error when policy permits continuation. - * - ``continue_on_error`` - - bool - - optional; default: ``false`` - - Per-step error policy. - * - ``input_mapping`` - - dict - - optional; default: ``{}`` - - Named input sources. - * - ``output_mapping`` - - dict - - optional; default: ``{}`` - - Named verdicts and destinations. - -Registered step types ---------------------- - -.. _recipe-step-pythonmodulestep: - -PythonModuleStep -~~~~~~~~~~~~~~~~ - -Calls a method or reads/writes an attribute in a Python module. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``action_type`` - - str - - required; allowed: ``method``, ``read_attribute``, ``write_attribute`` - - Operation performed on the Python module. - * - ``module`` - - str - - required - - Python module path. - * - ``method_name`` - - str - - optional - - Method name required by method actions. - -Canonical example: - -.. code-block:: yaml - - steptype: PythonModuleStep - step_name: Run test - description: Run a Python test method. - skip: false - critical: false - continue_on_error: false - action_type: method - module: tests.py - method_name: run - input_mapping: {} - output_mapping: {} - -.. _recipe-step-sequencestep: - -SequenceStep -~~~~~~~~~~~~ - -Runs another sequence as a step. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``sequence`` - - dict - - required - - Internal sequence reference. - -Canonical example: - -.. code-block:: yaml - - steptype: SequenceStep - step_name: Run calibration - description: Run an internal sequence. - skip: false - critical: false - continue_on_error: false - sequence: - type: internal - name: Calibration - input_mapping: {} - output_mapping: {} - -.. _recipe-step-userinteractionstep: - -UserInteractionStep -~~~~~~~~~~~~~~~~~~~ - -Displays an operator interaction prompt. - -Canonical example: - -.. code-block:: yaml - - steptype: UserInteractionStep - step_name: Confirm - description: Ask the operator to confirm. - skip: false - critical: false - continue_on_error: false - input_mapping: - message: - type: direct - value: Continue? - output_mapping: - output: - type: passfail - -.. _recipe-step-waitstep: - -WaitStep -~~~~~~~~ - -Waits for a non-negative duration in seconds. - -Required input names: ``wait_time``. - -Canonical example: - -.. code-block:: yaml - - steptype: WaitStep - step_name: Stabilize - description: Wait for hardware stabilization. - skip: false - critical: false - continue_on_error: false - input_mapping: - wait_time: - type: direct - value: 1 - output_mapping: {} - -.. _recipe-step-userloadingstep: - -UserLoadingStep -~~~~~~~~~~~~~~~ - -Prompts the operator to select a file. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``file_save_location`` - - dict - - optional - - Local or global destination for the selected file. - -Canonical example: - -.. code-block:: yaml - - steptype: UserLoadingStep - step_name: Load configuration - description: Ask the operator for a file. - skip: false - critical: false - continue_on_error: false - input_mapping: - message: - type: direct - value: Choose a file - output_mapping: - output: - type: passfail - -.. _recipe-step-userrunmethodstep: - -UserRunMethodStep -~~~~~~~~~~~~~~~~~ - -Optionally runs a Python method after an operator response. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``trigger_response`` - - str or list or dict - - optional - - Operator response that triggers execution. - * - ``action_type`` - - str - - optional - - Optional Python action type. - * - ``module`` - - str - - optional - - Optional Python module path. - * - ``method_name`` - - str - - optional - - Optional Python method name. - -Canonical example: - -.. code-block:: yaml - - steptype: UserRunMethodStep - step_name: Run calibration - description: Run on operator confirmation. - skip: false - critical: false - continue_on_error: false - trigger_response: run - action_type: method - module: tests.py - method_name: calibrate - input_mapping: {} - output_mapping: - output: - type: passfail - -.. _recipe-step-userwritestep: - -UserWriteStep -~~~~~~~~~~~~~ - -Writes an operator-provided value to a configured destination. - -Canonical example: - -.. code-block:: yaml - - steptype: UserWriteStep - step_name: Enter value - description: Ask the operator for a value. - skip: false - critical: false - continue_on_error: false - input_mapping: - message: - type: direct - value: Enter value - output_mapping: - output: - type: local - local_name: value - -.. _recipe-step-serialnumberstep: - -SerialNumberStep -~~~~~~~~~~~~~~~~ - -Captures the device serial number. - -Canonical example: - -.. code-block:: yaml - - steptype: SerialNumberStep - step_name: Scan serial number - description: Capture the device serial number. - skip: false - critical: false - continue_on_error: false - input_mapping: {} - output_mapping: {} - -.. _recipe-step-sshconnectstep: - -SSHConnectStep -~~~~~~~~~~~~~~ - -Opens the SSH client stored in recipe globals. - -Canonical example: - -.. code-block:: yaml - - steptype: SSHConnectStep - step_name: Connect - description: Open the SSH connection. - skip: false - critical: false - continue_on_error: false - input_mapping: {} - output_mapping: {} - -.. _recipe-step-sshclosestep: - -SSHCloseStep -~~~~~~~~~~~~ - -Closes the SSH client stored in recipe globals. - -Canonical example: - -.. code-block:: yaml - - steptype: SSHCloseStep - step_name: Disconnect - description: Close the SSH connection. - skip: false - critical: false - continue_on_error: false - input_mapping: {} - output_mapping: {} - -.. _recipe-step-sshuploadstep: - -SSHUploadStep -~~~~~~~~~~~~~ - -Uploads files through an SSH connection. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``files`` - - list - - required - - Local and remote file pairs to upload. - * - ``permissions`` - - int or str - - optional - - Optional remote permissions. - * - ``skip_if_sha256_match`` - - bool - - optional; default: ``false`` - - Skip files whose remote checksum matches. - * - ``local_package`` - - str - - optional - - Optional package containing local resources. - -Canonical example: - -.. code-block:: yaml - - steptype: SSHUploadStep - step_name: Deploy - description: Upload a file to the target. - skip: false - critical: false - continue_on_error: false - files: - - local: bin/tool - remote: /tmp/tool - input_mapping: {} - output_mapping: - passed: - type: passfail - -Input mapping types -------------------- - -.. _recipe-input-direct: - -**``direct``** - -Provides a literal value. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - optional; default: ``direct``; allowed: ``direct`` - - Input source type. - * - ``value`` - - any - - required - - Literal input value. - * - ``indexed`` - - bool - - optional; default: ``false`` - - Expand a list into indexed steps. - -Canonical mapping: - -.. code-block:: yaml - - type: direct - value: 1 - -.. _recipe-input-local: - -**``local``** - -Reads a sequence-local variable. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``local`` - - Input source type. - * - ``local_name`` - - str - - required - - Local variable name. - * - ``indexed`` - - bool - - optional; default: ``false``; legacy - - Compatibility field; only false is accepted. - -Canonical mapping: - -.. code-block:: yaml - - type: local - local_name: local_value - -.. _recipe-input-global: - -**``global``** - -Reads a recipe-global variable. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``global`` - - Input source type. - * - ``global_name`` - - str - - required - - Global variable name. - * - ``indexed`` - - bool - - optional; default: ``false``; legacy - - Compatibility field; only false is accepted. - -Canonical mapping: - -.. code-block:: yaml - - type: global - global_name: global_value - -.. _recipe-input-method: - -**``method``** - -Resolves a method reference for the step. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``method`` - - Input source type. - * - ``value`` - - any - - required - - Method reference. - * - ``indexed`` - - bool - - optional; default: ``false``; legacy - - Compatibility field; only false is accepted. - -Canonical mapping: - -.. code-block:: yaml - - type: method - value: helper - -Output mapping types --------------------- - -.. _recipe-output-passfail: - -**``passfail``** - -Interprets the output as a pass/fail verdict. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``passfail`` - - Output mapping type. - -Canonical mapping: - -.. code-block:: yaml - - type: passfail - -.. _recipe-output-equals: - -**``equals``** - -Passes when the output equals the configured value. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``equals`` - - Output mapping type. - * - ``value`` - - any - - required - - Expected value. - -Canonical mapping: - -.. code-block:: yaml - - type: equals - value: 3 - -.. _recipe-output-range: - -**``range``** - -Passes when the output is within an inclusive range. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``range`` - - Output mapping type. - * - ``min`` - - any - - required - - Minimum accepted value. - * - ``max`` - - any - - required - - Maximum accepted value. - -Canonical mapping: - -.. code-block:: yaml - - type: range - min: 1 - max: 4 - -.. _recipe-output-passthrough: - -**``passthrough``** - -Uses the nested result without adding a verdict. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``passthrough`` - - Output mapping type. - -Canonical mapping: - -.. code-block:: yaml - - type: passthrough - -.. _recipe-output-local: - -**``local``** - -Stores the output in a sequence-local variable. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``local`` - - Output mapping type. - * - ``local_name`` - - str - - required - - Local destination variable. - -Canonical mapping: - -.. code-block:: yaml - - type: local - local_name: saved - -.. _recipe-output-global: - -**``global``** - -Stores the output in a recipe-global variable. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``global`` - - Output mapping type. - * - ``global_name`` - - str - - required - - Global destination variable. - -Canonical mapping: - -.. code-block:: yaml - - type: global - global_name: saved - -.. _recipe-output-image: - -**``image``** - -Publishes an image output for presentation. - -.. list-table:: - :header-rows: 1 - :widths: 18 14 28 40 - - * - Field - - Type - - Requirement - - Description - * - ``type`` - - str - - required; allowed: ``image`` - - Output mapping type. - -Canonical mapping: - -.. code-block:: yaml - - type: image - -Semantic constraints --------------------- - -* ``parser`` — Sources must be readable text and safe, non-recursive YAML without duplicate keys. - Diagnostics: ``invalid-source``, ``file-read-error``, ``yaml-syntax-error``, ``yaml-construction-error``, ``unsafe-yaml``, ``recursive-alias``, ``duplicate-key``. -* ``documents and steps`` — Only declared fields, field types, and allowed values are accepted. - Diagnostics: ``missing-field``, ``invalid-field-type``, ``invalid-field-value``, ``unknown-field``, ``unsupported-recipe-version``, ``invalid-report-mode``, ``invalid-action-type``. -* ``input and output mappings`` — Mapping variants accept only their declared fields and require their declared values. - Diagnostics: ``invalid-input-mapping``, ``unknown-input-source``, ``unknown-input-field``, ``invalid-input-field-type``, ``missing-input-source-value``, ``invalid-output-mapping``, ``unknown-output-type``, ``unknown-output-field``, ``invalid-output-field-type``, ``missing-output-field``. -* ``recipe`` — A recipe is safe multi-document YAML with one header followed by at least one sequence. - Diagnostics: ``empty-recipe``, ``invalid-header``, ``invalid-sequence``. -* ``step`` — Every step is a mapping with a registered step type. - Diagnostics: ``invalid-step``, ``unknown-step-type``. -* ``sequence`` — Sequence names are unique and main_sequence names an existing sequence. - Diagnostics: ``duplicate-sequence``, ``unknown-main-sequence``. -* ``SequenceStep`` — Only internal references are accepted and the named target sequence must exist. - Diagnostics: ``invalid-sequence-reference``, ``unknown-sequence-reference``. -* ``PythonModuleStep`` — A method action requires method_name. - Diagnostics: ``missing-method-name``. -* ``UserLoadingStep`` — A file destination names a local or global variable. - Diagnostics: ``invalid-file-save-location``. -* ``input mapping`` — Indexed inputs are direct lists and all indexed lists on a step have equal length. - Diagnostics: ``invalid-indexed-flag``, ``invalid-indexed-input``, ``unequal-indexed-inputs``. -* ``output mapping`` — passthrough must be the only verdict mapping on its step. - Diagnostics: ``mixed-passthrough``. -* ``step`` — Step-specific required input names must be present. - Diagnostics: ``missing-required-input``, ``missing-input-mapping``. -* ``SSH steps`` — SSH steps require connection globals; setup connections require teardown closure, and uploads require an earlier connection. - Diagnostics: ``missing-ssh-global``, ``missing-ssh-credential``, ``missing-ssh-connect``, ``missing-ssh-close``. -* ``step`` — IndexedStep is runtime-generated and cannot be authored in recipe YAML. - Diagnostics: ``internal-step-type``. -* ``sequence`` — serial_number is accepted with a warning and omitted from the typed model. - Diagnostics: ``legacy-sequence-field``. -* ``step and input mapping`` — Noncanonical step casing and omitted direct input types are normalized with warnings. - Diagnostics: ``noncanonical-step-type``, ``implicit-direct-input``. - -Canonicalization ----------------- - -The parser normalizes step type casing, implicit direct inputs, mapping -defaults, and optional flags. Canonical serialization uses explicit YAML -document starts and stable field ordering. Comments and original formatting -are not preserved; parse/dump/reparse model equality is the guarantee. - -``IndexedStep`` is reserved for runtime construction and cannot be authored -as a recipe step. diff --git a/docs/source/index.rst b/docs/source/index.rst index 59b44d1..b601a39 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,6 +57,7 @@ Documentation contents api architecture recipe_language_architecture + recipe_language_maintenance _generated/recipe_language_reference gui_architecture yaml_format diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index 565460c..af8488c 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -243,7 +243,7 @@ The runtime registry remains because a canonical discriminator such as ``PythonModuleStep`` must be associated with the Python class that implements its behavior. It is a behavior registry, not a second language schema: field names, types, defaults, and structural rules remain exclusively in the -Pydantic models. A completeness test requires every authorable model +Pydantic models. A completeness test requires every step-definition model discriminator to have exactly one executable implementation. Concrete ``_step()`` methods in ``steps.py`` continue to own execution. For @@ -251,7 +251,7 @@ example, the executable ``PythonModuleStep`` still imports and invokes Python code; it no longer validates an untrusted recipe dictionary. Common definition fields are dumped once by ``Step.build_step()`` and passed to the existing constructors. ``IndexedStep`` remains a runtime-generated wrapper and is -never added to the authorable model union. +never added to the ``StepDefinition`` model union. Synthetic runtime operations are also constructed directly. For example, ``Recipe.run()`` must not fabricate a recipe dictionary merely to execute the @@ -276,6 +276,10 @@ production parser. It is not a bundled recipe. Maintaining the documentation ----------------------------- +See :doc:`recipe_language_maintenance` for the complete extension and version +upgrade workflow, including the definition/runtime boundary and the purpose of +``STEP_TYPE_REGISTRY``. + Every Sphinx build generates both artifacts before reading documentation sources:: @@ -299,13 +303,6 @@ Pydantic is a core production dependency. CI and any documentation build image must install the ``doc`` extra and include the model, schema generator, and JSON-only renderer sources. -When adding a step or mapping, update its Pydantic model and discriminated -union, add an independent round-trip fixture, and run the documentation build. -Review the generated schema and reference in the build output when relevant. -Add custom semantic code only when a rule requires document, sibling, or -ordering context. Handwritten architecture prose explains those relationships; -it must link to generated fields rather than restating field tables. - The documentation contract is protected by tests that generate into temporary directories, verify deterministic model-to-JSON and JSON-to-RST output, count all discriminator variants, validate the example, check literal-include diff --git a/docs/source/recipe_language_maintenance.rst b/docs/source/recipe_language_maintenance.rst new file mode 100644 index 0000000..5d421eb --- /dev/null +++ b/docs/source/recipe_language_maintenance.rst @@ -0,0 +1,125 @@ +.. SPDX-FileCopyrightText: 2026 CERN +.. +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Maintaining the Recipe Language +=============================== + +This page is for maintainers extending recipe language ``2.0.0`` or preparing +a future language version. For the current data flow and validation stages, +see :doc:`recipe_language_architecture`. Exact fields and variants are listed +in the :doc:`_generated/recipe_language_reference`. + +Terminology and ownership +------------------------- + +``Step definition`` + Declarative recipe data represented by a strict, frozen Pydantic model in + ``pypts.recipe_language``. ``StepDefinition`` is the discriminated union of + every step users may place in YAML. + +``Validated recipe definition`` + The aggregate ``pypts.recipe_language.Recipe`` returned by + ``ParseResult.require_recipe()`` after YAML, structural, and semantic + validation succeeds. + +``Executable step`` + A runtime object implemented in ``pypts.steps``. It owns behavior such as + ``_step()`` but does not define the authored recipe schema. + +``Runtime-only step`` + An internal operation that cannot be written directly in YAML. For example, + ``IndexedStep`` is created when a validated direct input has ``indexed`` set. + +The Pydantic models are the only structural definition of the language. JSON +Schema, generated reference documentation, YamVIEW selectors, and form fields +all derive from them. Do not add a second list of supported steps, mappings, +required fields, defaults, or aliases in a consumer. + +Runtime dispatch and dependency direction +----------------------------------------- + +``Step.build_step()`` in ``pypts.recipe`` is the single boundary between a +validated step definition and an executable step:: + + recipe_language.StepDefinition + | + v + Step.build_step() + | + v + STEP_TYPE_REGISTRY + | + v + executable class in steps.py + +``STEP_TYPE_REGISTRY`` is intentionally beside the factory. It answers only: +"Which executable class implements this validated discriminator?" It does not +own fields, defaults, validation rules, or the set exposed by JSON Schema. + +Moving the registry into ``steps.py`` would reverse the existing dependency +direction: executable classes in ``steps.py`` derive from or interact with +runtime types defined in ``recipe.py``. Keeping dispatch in ``recipe.py`` avoids +introducing a runtime adapter or circular registration mechanism. A +completeness test requires the registry keys to equal the discriminators in +``STEP_DEFINITION_MODELS`` exactly. + +Adding a step definition +------------------------ + +1. Add the strict Pydantic definition in ``recipe_language.py`` and include it + in the ``StepDefinition`` discriminated union. Use the exact canonical + ``steptype`` literal; do not add normalization or lowercase aliases. +2. Implement the executable class in ``steps.py``. Keep recipe-structure + validation in the definition model or parser rather than its constructor. +3. Add one discriminator-to-class entry to ``STEP_TYPE_REGISTRY`` beside + ``Step.build_step()``. Do not register runtime-only wrappers. +4. Add parse/dump/reparse coverage, runtime construction assertions, and a + behavioral execution test. The registry-completeness test must remain exact. +5. Build the documentation and review the generated schema, reference section, + and YamVIEW selector. These consumers should update without hardcoded lists. + +Adding fields or mapping variants +--------------------------------- + +Put a field's type, required status, default, description, examples, aliases, +and serialization behavior on its Pydantic declaration. A rule concerning one +object belongs in a model validator. A rule requiring other sequences, sibling +values, reference resolution, or execution order belongs in the parser's +aggregate semantic pass so it can produce a structured source-span diagnostic. + +For a new input or output mapping, add its definition to the corresponding +discriminated union and test parse, canonical dump, reparse, generated schema, +reference rendering, and YamVIEW mapping rows. Runtime mapping behavior must +also receive a focused execution test. + +Preparing a language version upgrade +------------------------------------ + +A version change is a deliberate compatibility decision, not a normalization +shortcut. Before changing the header's ``recipe_version`` literal: + +* define the supported version policy and whether migration is a separate + release phase; +* document removed, renamed, and newly required fields and discriminators; +* update parser diagnostics for missing, legacy, and unsupported versions; +* update canonical examples, fixtures, schema expectations, and release notes; +* decide explicitly whether multiple versions have separate models and runtime + paths; never silently coerce one version into another; +* verify that invalid or legacy documents cannot construct executable state; +* review generated JSON Schema and reference artifacts before release. + +Documentation and verification +------------------------------ + +Sphinx generates the JSON Schema first and renders the human reference from +that exact file. Do not hand-edit files below ``docs/source/_generated``. Run:: + + python -m pypts.recipe_artifacts + python -m pytest tests + python -m ruff check src tests + python -m sphinx -W -b html docs/source docs/_build/html + +At minimum, a recipe-language change must pass deterministic artifact tests, +all discriminator and runtime-registry completeness tests, the full unit and +functional suite, and Sphinx with warnings treated as errors. diff --git a/src/pypts/YamVIEW/recipe_step_setup.py b/src/pypts/YamVIEW/recipe_step_setup.py index a94030f..f7a30fa 100644 --- a/src/pypts/YamVIEW/recipe_step_setup.py +++ b/src/pypts/YamVIEW/recipe_step_setup.py @@ -15,7 +15,7 @@ from pydantic import TypeAdapter from pypts.recipe_language import Recipe as RecipeDefinition -from pypts.recipe_language import Step as AuthorableStepDefinition +from pypts.recipe_language import StepDefinition def recipe_form_schema() -> dict[str, Any]: @@ -83,7 +83,7 @@ def variants(name: str): return result return { - "steps": variants("Step"), + "steps": variants("StepDefinition"), "inputs": variants("InputMapping"), "outputs": variants("OutputMapping"), } @@ -142,7 +142,7 @@ def __init__(self, variant: dict[str, Any], parent=None): layout.addWidget(widget) def values(self) -> dict[str, Any]: - """Read authorable values from the schema-selected controls.""" + """Read recipe step-definition values from the schema controls.""" values = {} fields = self.variant["fields"] for name, widget in self.field_widgets.items(): @@ -513,7 +513,7 @@ def __init__(self,use_input_mapping=True, use_output_mapping=True, parent=None): layout.addWidget(QLabel("Steptype")) self.list_steptype = QComboBox() - self.steptypes = list(discriminator_schemas("Step")) + self.steptypes = list(discriminator_schemas("StepDefinition")) self.list_steptype.addItems(self.steptypes) layout.addWidget(self.list_steptype) @@ -580,7 +580,7 @@ def on_step_type_changed(self, step_type: str): self._render_schema_step(step_type) def _render_schema_step(self, step_type: str): - """Render the selected authorable step directly from JSON Schema.""" + """Render the selected step definition directly from JSON Schema.""" self.schema_form = SchemaFormWidget(self.form_description["steps"][step_type]) self.step_specific_container.addWidget(self.schema_form) self.step_specific_container.addStretch() @@ -951,9 +951,11 @@ def accept(self): StepID = str(uuid.uuid4()) try: - authorable = self.schema_form.values() - authorable["steptype"] = step_type - definition = TypeAdapter(AuthorableStepDefinition).validate_python(authorable) + step_definition_data = self.schema_form.values() + step_definition_data["steptype"] = step_type + definition = TypeAdapter(StepDefinition).validate_python( + step_definition_data + ) except Exception as error: self.setStyleSheet("""QMessageBox QPushButton { color: black;}""") QMessageBox.warning(self, "Invalid step", str(error)) diff --git a/src/pypts/recipe.py b/src/pypts/recipe.py index 812f5ef..e6fc6e7 100644 --- a/src/pypts/recipe.py +++ b/src/pypts/recipe.py @@ -19,7 +19,7 @@ from threading import Event from pypts.utils import WAIT_FOR_TERMINATION from pypts.recipe_language import ( - CommonStep as AuthorableStepDefinition, + CommonStepDefinition as ValidatedStepDefinition, DirectInput as DirectInputDefinition, Recipe as RecipeDefinition, Sequence as SequenceDefinition, @@ -655,18 +655,20 @@ def run_steps(runtime: Runtime, step_list: List[Self], parent_step: uuid.UUID, s return step_results # aggregate_result # single pass or fail type @staticmethod - def build_step(step_definition: AuthorableStepDefinition): - """ - This function translates from validated Pydantic step definitions - to the corresponding executable Step objects. + def build_step(step_definition: ValidatedStepDefinition): + """Build an executable step from a validated recipe step definition. + + This is the single boundary between declarative recipe data and + executable runtime behavior. The definition has already passed the + Pydantic and aggregate semantic validation performed by the parser. Args: - step_definition (AuthorableStepDefinition): the validated step definition + step_definition: Validated Pydantic step definition. Returns: - Step: This is a fully configured step object + Fully configured executable :class:`Step`. """ - if not isinstance(step_definition, AuthorableStepDefinition): + if not isinstance(step_definition, ValidatedStepDefinition): raise TypeError("Step.build_step requires a validated step definition") step_type = step_definition.steptype step_class = STEP_TYPE_REGISTRY.get(step_type) @@ -714,7 +716,13 @@ def build_step(step_definition: AuthorableStepDefinition): SSHUploadStep as ExecutableSSHUploadStep, ) -# This dictionary maps step type names to their corresponding executable classes, allowing dynamic instantiation based on the step type specified in the recipe. +# Runtime behavior dispatch belongs beside Step.build_step(), the sole typed +# factory. This registry does not define fields or supported recipe structures: +# those come exclusively from recipe_language.StepDefinition's discriminated +# union. It only binds each already-validated canonical discriminator to the +# concrete class in steps.py that implements its behavior. Keeping the mapping +# here also preserves the dependency direction because steps.py depends on +# runtime types defined in this module. STEP_TYPE_REGISTRY = { "PythonModuleStep": ExecutablePythonModuleStep, "SequenceStep": ExecutableSequenceStep, diff --git a/src/pypts/recipe_language.py b/src/pypts/recipe_language.py index 6544289..b135e82 100644 --- a/src/pypts/recipe_language.py +++ b/src/pypts/recipe_language.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2026 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -"""Authoritative Pydantic model for the candidate recipe language. +"""Authoritative Pydantic definitions for recipe language 2.0.0. Field declarations intentionally own types, defaults, descriptions, examples, serialization behavior, and JSON Schema. There is no parallel field registry. @@ -22,7 +22,7 @@ def described(description: str, *, example: Any = None, **kwargs: Any) -> Any: class RecipeModel(BaseModel): - """Strict, immutable base for all authorable structures.""" + """Strict, immutable base for every validated recipe definition.""" model_config = ConfigDict( extra="forbid", @@ -161,8 +161,8 @@ class UploadFile(RecipeModel): remote: str = described("Remote destination path.", example="/tmp/tool") -class CommonStep(RecipeModel): - """Fields shared by every authorable step.""" +class CommonStepDefinition(RecipeModel): + """Fields shared by every recipe step definition.""" step_name: str = described("Human-readable step name.", example="Run test") description: str = described("Purpose of the step.", example="Run a test operation.") @@ -180,7 +180,7 @@ class CommonStep(RecipeModel): ) -class PythonModuleStep(CommonStep): +class PythonModuleStep(CommonStepDefinition): """Calls a method or reads/writes an attribute in a Python module.""" steptype: Literal["PythonModuleStep"] = described( @@ -203,7 +203,7 @@ def method_actions_have_names(self) -> PythonModuleStep: # docs:method-name-end -class SequenceStep(CommonStep): +class SequenceStep(CommonStepDefinition): """Runs another sequence as a step.""" steptype: Literal["SequenceStep"] = described( @@ -214,7 +214,7 @@ class SequenceStep(CommonStep): ) -class UserInteractionStep(CommonStep): +class UserInteractionStep(CommonStepDefinition): """Displays an operator interaction prompt.""" steptype: Literal["UserInteractionStep"] = described( @@ -222,7 +222,7 @@ class UserInteractionStep(CommonStep): ) -class WaitStep(CommonStep): +class WaitStep(CommonStepDefinition): """Waits for a non-negative duration in seconds.""" steptype: Literal["WaitStep"] = described("Canonical registered step type.", example="WaitStep") @@ -236,7 +236,7 @@ def has_wait_time(self) -> WaitStep: # docs:wait-time-end -class UserLoadingStep(CommonStep): +class UserLoadingStep(CommonStepDefinition): """Prompts the operator to select a file.""" steptype: Literal["UserLoadingStep"] = described( @@ -249,7 +249,7 @@ class UserLoadingStep(CommonStep): ) -class UserRunMethodStep(CommonStep): +class UserRunMethodStep(CommonStepDefinition): """Optionally runs a Python method after an operator response.""" steptype: Literal["UserRunMethodStep"] = described( @@ -263,7 +263,7 @@ class UserRunMethodStep(CommonStep): method_name: str | None = described("Optional Python method name.", example="run", default=None) -class UserWriteStep(CommonStep): +class UserWriteStep(CommonStepDefinition): """Writes an operator-provided value to a configured destination.""" steptype: Literal["UserWriteStep"] = described( @@ -271,7 +271,7 @@ class UserWriteStep(CommonStep): ) -class SerialNumberStep(CommonStep): +class SerialNumberStep(CommonStepDefinition): """Captures the device serial number.""" steptype: Literal["SerialNumberStep"] = described( @@ -279,7 +279,7 @@ class SerialNumberStep(CommonStep): ) -class SSHConnectStep(CommonStep): +class SSHConnectStep(CommonStepDefinition): """Opens the SSH client stored in recipe globals.""" steptype: Literal["SSHConnectStep"] = described( @@ -287,7 +287,7 @@ class SSHConnectStep(CommonStep): ) -class SSHCloseStep(CommonStep): +class SSHCloseStep(CommonStepDefinition): """Closes the SSH client stored in recipe globals.""" steptype: Literal["SSHCloseStep"] = described( @@ -295,7 +295,7 @@ class SSHCloseStep(CommonStep): ) -class SSHUploadStep(CommonStep): +class SSHUploadStep(CommonStepDefinition): """Uploads files through an SSH connection.""" steptype: Literal["SSHUploadStep"] = described( @@ -316,7 +316,7 @@ class SSHUploadStep(CommonStep): ) -type Step = Annotated[ +type StepDefinition = Annotated[ PythonModuleStep | SequenceStep | UserInteractionStep @@ -365,9 +365,9 @@ class Sequence(RecipeModel): parameters: dict[str, Any] = described("Reserved sequence input metadata.", example={}) outputs: dict[str, Any] = described("Reserved sequence output metadata.", example={}) locals: dict[str, Any] = described("Variables local to the sequence.", example={}) - setup_steps: list[Step] = described("Steps run before the main steps.", example=[]) - steps: list[Step] = described("Ordered main steps.", example=[]) - teardown_steps: list[Step] = described("Steps run during teardown.", example=[]) + setup_steps: list[StepDefinition] = described("Steps run before the main steps.", example=[]) + steps: list[StepDefinition] = described("Ordered main steps.", example=[]) + teardown_steps: list[StepDefinition] = described("Steps run during teardown.", example=[]) class Recipe(RecipeModel): @@ -384,6 +384,6 @@ def _union_models(annotation: Any) -> tuple[type[RecipeModel], ...]: return get_args(annotated_union) -STEP_MODELS = _union_models(Step) +STEP_DEFINITION_MODELS = _union_models(StepDefinition) INPUT_MODELS = _union_models(InputMapping) OUTPUT_MODELS = _union_models(OutputMapping) diff --git a/src/pypts/recipe_reference.py b/src/pypts/recipe_reference.py index 823b0ee..139d81b 100644 --- a/src/pypts/recipe_reference.py +++ b/src/pypts/recipe_reference.py @@ -149,7 +149,7 @@ def _common_step_fields( def render_reference(schema: dict[str, Any]) -> str: """Render deterministic RST using only a parsed JSON Schema document.""" definitions = schema["$defs"] - steps = _discriminator_mapping(definitions, "Step") + steps = _discriminator_mapping(definitions, "StepDefinition") inputs = _discriminator_mapping(definitions, "InputMapping") outputs = _discriminator_mapping(definitions, "OutputMapping") common_fields = _common_step_fields(definitions, list(steps.values())) @@ -171,7 +171,8 @@ def render_reference(schema: dict[str, Any]) -> str: ":download:`Download the JSON Schema `.", "", "See :doc:`/recipe_language_architecture` for parsing, semantic rules,", - "documentation maintenance, and the planned YamVIEW and sequencer flows.", + "YamVIEW, and runtime construction. Maintainers should also read", + ":doc:`/recipe_language_maintenance`.", "", "Documents", "---------", @@ -189,7 +190,7 @@ def render_reference(schema: dict[str, Any]) -> str: representative = next(iter(steps.values())) lines.extend(_field_table(definitions[representative], include=common_fields)) - lines.extend(("Authorable steps", "----------------", "")) + lines.extend(("Step definitions", "----------------", "")) for discriminator, definition_name in steps.items(): lines.extend(_model_section( definition_name, diff --git a/tests/unit_tests/test_recipe.py b/tests/unit_tests/test_recipe.py index e3abee6..caab55c 100644 --- a/tests/unit_tests/test_recipe.py +++ b/tests/unit_tests/test_recipe.py @@ -16,9 +16,8 @@ Runtime, Step, ) -from pypts.recipe_language import STEP_MODELS +from pypts.recipe_language import STEP_DEFINITION_MODELS, StepDefinition from pypts.recipe_language import Recipe as RecipeDefinition -from pypts.recipe_language import Step as AuthorableStep from pypts.recipe_parser import RecipeParseError, dump_recipe @@ -152,17 +151,17 @@ def test_recipe_path_requires_v2_and_raises_structured_error(tmp_path): assert "unsupported-recipe-version" in {d.code for d in caught.value.diagnostics} -def test_registry_exactly_matches_authorable_discriminators(): +def test_registry_exactly_matches_step_definition_discriminators(): discriminators = { model.model_fields["steptype"].examples[0] - for model in STEP_MODELS + for model in STEP_DEFINITION_MODELS } assert set(STEP_TYPE_REGISTRY) == discriminators @pytest.mark.parametrize("name", STEP_EXAMPLES) def test_every_typed_definition_builds_its_concrete_executable(name): - typed = TypeAdapter(AuthorableStep).validate_python(STEP_EXAMPLES[name]) + typed = TypeAdapter(StepDefinition).validate_python(STEP_EXAMPLES[name]) executable = Step.build_step(typed) assert type(executable).__name__ == name assert executable.input_mapping == typed.model_dump( diff --git a/tests/unit_tests/test_recipe_language.py b/tests/unit_tests/test_recipe_language.py index f65e3d5..08a29d1 100644 --- a/tests/unit_tests/test_recipe_language.py +++ b/tests/unit_tests/test_recipe_language.py @@ -12,12 +12,17 @@ import yaml from pydantic import ValidationError +from pypts import recipe_language from pypts.recipe_artifacts import ( DEFAULT_REFERENCE_PATH, DEFAULT_SCHEMA_PATH, render_json_schema, ) -from pypts.recipe_language import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS +from pypts.recipe_language import ( + INPUT_MODELS, + OUTPUT_MODELS, + STEP_DEFINITION_MODELS, +) from pypts.recipe_parser import ( RecipeParseError, dump_recipe, @@ -30,6 +35,14 @@ RECIPES = ROOT / "src" / "pypts" / "recipes" +def test_step_definition_names_are_the_only_public_union_and_model_registry(): + assert hasattr(recipe_language, "StepDefinition") + assert hasattr(recipe_language, "STEP_DEFINITION_MODELS") + assert not hasattr(recipe_language, "Step") + assert not hasattr(recipe_language, "CommonStep") + assert not hasattr(recipe_language, "STEP_MODELS") + + def header(**updates): value = { "name": "Pydantic spike", @@ -130,7 +143,9 @@ def recipe_for_step(step): return source(*documents) -@pytest.mark.parametrize("model", STEP_MODELS, ids=lambda model: model.__name__) +@pytest.mark.parametrize( + "model", STEP_DEFINITION_MODELS, ids=lambda model: model.__name__ +) def test_every_step_validates_serializes_and_reparses(model): first = parse_recipe_text(recipe_for_step(STEP_EXAMPLES[model.__name__])) assert first.is_valid, first.errors @@ -375,14 +390,20 @@ def test_generated_schema_and_reference_are_complete_and_current(): assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") assert reference == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") definitions = schema["$defs"] - for model in STEP_MODELS + INPUT_MODELS + OUTPUT_MODELS: + for model in STEP_DEFINITION_MODELS + INPUT_MODELS + OUTPUT_MODELS: assert model.__name__ in definitions kind = model.model_fields.get("steptype") or model.model_fields["type"] anchor_kind = kind.examples[0].lower() - group = "step" if model in STEP_MODELS else "input" if model in INPUT_MODELS else "output" + group = ( + "step" + if model in STEP_DEFINITION_MODELS + else "input" + if model in INPUT_MODELS + else "output" + ) assert reference.count(f".. _recipe-v2-{group}-{anchor_kind}:") == 1 - report = STEP_MODELS[0].model_fields["skip"] + report = STEP_DEFINITION_MODELS[0].model_fields["skip"] assert report.description in reference assert 'default ``false``' in reference assert 'Example: ``false``.' in reference diff --git a/tests/unit_tests/test_recipe_pydantic_docs.py b/tests/unit_tests/test_recipe_pydantic_docs.py index 1c477fd..57a3b52 100644 --- a/tests/unit_tests/test_recipe_pydantic_docs.py +++ b/tests/unit_tests/test_recipe_pydantic_docs.py @@ -24,6 +24,7 @@ ROOT = Path(__file__).parents[2] DOC_RECIPE = ROOT / "docs" / "source" / "_examples" / "recipe_v2.yml" ARCHITECTURE = ROOT / "docs" / "source" / "recipe_language_architecture.rst" +MAINTENANCE = ROOT / "docs" / "source" / "recipe_language_maintenance.rst" REFERENCE_RENDERER = ROOT / "src" / "pypts" / "recipe_reference.py" @@ -58,7 +59,7 @@ def test_every_discriminator_is_rendered_once(): schema_text, reference = rendered_artifacts() schema = json.loads(schema_text) for group, definition in ( - ("step", "Step"), + ("step", "StepDefinition"), ("input", "InputMapping"), ("output", "OutputMapping"), ): @@ -67,6 +68,12 @@ def test_every_discriminator_is_rendered_once(): assert reference.count(anchor) == 1 +def test_schema_uses_step_definition_name_without_legacy_alias(): + schema = json.loads(render_json_schema()) + assert "StepDefinition" in schema["$defs"] + assert "Step" not in schema["$defs"] + + def test_reference_metadata_comes_from_json_schema(): schema_text, reference = rendered_artifacts() schema = json.loads(schema_text) @@ -144,5 +151,10 @@ def test_sphinx_sources_link_reference_schema_and_example(): index = (ROOT / "docs" / "source" / "index.rst").read_text(encoding="utf-8") architecture = ARCHITECTURE.read_text(encoding="utf-8") assert "_generated/recipe_language_reference" in index + assert "recipe_language_maintenance" in index assert "_generated/recipe_language.schema.json" in architecture assert "_examples/recipe_v2.yml" in architecture + maintenance = MAINTENANCE.read_text(encoding="utf-8") + assert "STEP_TYPE_REGISTRY" in maintenance + assert "STEP_DEFINITION_MODELS" in maintenance + assert "Preparing a language version upgrade" in maintenance diff --git a/tests/unit_tests/test_recipe_reference.py b/tests/unit_tests/test_recipe_reference.py index 42efaad..a2507d7 100644 --- a/tests/unit_tests/test_recipe_reference.py +++ b/tests/unit_tests/test_recipe_reference.py @@ -16,11 +16,15 @@ def test_reference_renders_every_discriminator_from_generated_json(): schema = json.loads(render_json_schema()) rendered = render_reference(schema) for group, definition in ( - ("step", "Step"), ("input", "InputMapping"), ("output", "OutputMapping") + ("step", "StepDefinition"), + ("input", "InputMapping"), + ("output", "OutputMapping"), ): mapping = schema["$defs"][definition]["discriminator"]["mapping"] for name in mapping: assert rendered.count(f".. _recipe-v2-{group}-{name.lower()}:") == 1 + assert "Step definitions" in rendered + assert "Authorable" not in rendered def test_json_only_renderer_has_no_model_runtime_gui_or_sphinx_imports(): diff --git a/tests/unit_tests/test_steps.py b/tests/unit_tests/test_steps.py index 923df7f..d64a181 100644 --- a/tests/unit_tests/test_steps.py +++ b/tests/unit_tests/test_steps.py @@ -424,7 +424,7 @@ def test_build_indexed_step_wraps_when_indexed(self): assert isinstance(step.template_step, WaitStep) def test_rejects_unvalidated_dictionary(self): - with pytest.raises(TypeError, match="validated authorable"): + with pytest.raises(TypeError, match="validated step definition"): Step.build_step({"steptype": "WaitStep"}) diff --git a/tests/unit_tests/test_yamview_schema.py b/tests/unit_tests/test_yamview_schema.py index 3428170..ef5c0e2 100644 --- a/tests/unit_tests/test_yamview_schema.py +++ b/tests/unit_tests/test_yamview_schema.py @@ -2,7 +2,11 @@ # SPDX-License-Identifier: LGPL-2.1-or-later """Schema-driven YamVIEW form tests.""" -from pypts.recipe_language import INPUT_MODELS, OUTPUT_MODELS, STEP_MODELS +from pypts.recipe_language import ( + INPUT_MODELS, + OUTPUT_MODELS, + STEP_DEFINITION_MODELS, +) from pypts.YamVIEW.recipe_step_setup import ( DiscriminatedMappingWidget, Step_setup, @@ -17,7 +21,9 @@ def discriminator(model): def test_form_selectors_and_metadata_come_from_production_schema(): description = recipe_form_description() - assert set(description["steps"]) == {discriminator(model) for model in STEP_MODELS} + assert set(description["steps"]) == { + discriminator(model) for model in STEP_DEFINITION_MODELS + } assert set(description["inputs"]) == {discriminator(model) for model in INPUT_MODELS} assert set(description["outputs"]) == {discriminator(model) for model in OUTPUT_MODELS} From 3db1bd9aaf39e180d92645251006eb54c9607f3d Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 11:10:12 +0200 Subject: [PATCH 10/14] migrate recipe examples to 2.0.0 language --- docs/source/_examples/device_recipe_v2.yml | 57 + .../source/_examples/instrument_recipe_v2.yml | 30 + docs/source/architecture.rst | 45 +- docs/source/instruments.rst | 26 +- docs/source/recipe_language_architecture.rst | 9 +- docs/source/report_generation.rst | 1 + docs/source/troubleshooting.rst | 83 +- docs/source/usage.rst | 138 +- docs/source/yaml_format.rst | 1184 ++--------------- spikes/recipe_pydantic/__init__.py | 29 - src/pypts/YamVIEW/customGUIModules.py | 4 +- src/pypts/YamVIEW/recipe_creator.py | 6 +- src/pypts/example_commented_recipes/readme.md | 5 - .../simple_multiplestep_recipe.yml | 164 --- .../Minimal_setup/Minimal_setup_recipe.yml | 12 +- .../Package_based_recipe.yml | 38 +- src/pypts/recipe_parser.py | 13 +- src/pypts/recipes/Instrument_test.yml | 4 +- src/pypts/recipes/RTM_recipe.yml | 2 +- .../recipes/RTM_recipe_shared_object.yml | 2 +- src/pypts/recipes/black_forest.yml | 18 +- src/pypts/recipes/comprehensive_recipe.yml | 6 +- src/pypts/recipes/graph_testing.yml | 8 +- src/pypts/recipes/simple_recipe.yml | 6 +- .../recipes/subsequence_executions_draft.yml | 94 -- tests/functional_tests/test_recipes_format.py | 2 +- .../test_environment_setup_recipes.py | 30 + tests/unit_tests/test_recipe.py | 32 +- tests/unit_tests/test_recipe.yaml | 88 -- tests/unit_tests/test_recipe_language.py | 78 +- tests/unit_tests/test_recipe_pydantic_docs.py | 14 +- tests/unit_tests/test_verify_recipe.py | 4 +- 32 files changed, 436 insertions(+), 1796 deletions(-) create mode 100644 docs/source/_examples/device_recipe_v2.yml create mode 100644 docs/source/_examples/instrument_recipe_v2.yml delete mode 100644 spikes/recipe_pydantic/__init__.py delete mode 100644 src/pypts/example_commented_recipes/readme.md delete mode 100644 src/pypts/example_commented_recipes/simple_multiplestep_recipe.yml delete mode 100644 src/pypts/recipes/subsequence_executions_draft.yml create mode 100644 tests/unit_tests/test_environment_setup_recipes.py delete mode 100644 tests/unit_tests/test_recipe.yaml diff --git a/docs/source/_examples/device_recipe_v2.yml b/docs/source/_examples/device_recipe_v2.yml new file mode 100644 index 0000000..b21daf3 --- /dev/null +++ b/docs/source/_examples/device_recipe_v2.yml @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: CC-BY-SA-4.0 +--- +name: Device acceptance recipe +version: "1.0" +recipe_version: 2.0.0 +description: Configure a device, measure it, and disconnect safely. +main_sequence: Main +test_package: my_project.tests +globals: + device_address: COM3 + test_voltage: 5.0 +--- +sequence_name: Main +description: Run the device acceptance flow. +parameters: {} +locals: + measurement: null +outputs: {} +setup_steps: + - steptype: PythonModuleStep + step_name: Configure device + description: Configure the device before measurement. + action_type: method + module: device_driver.py + method_name: setup_device + input_mapping: + port: {type: global, global_name: device_address} + voltage: {type: global, global_name: test_voltage} + output_mapping: + success: {type: passfail} +steps: + - steptype: WaitStep + step_name: Initial delay + description: Allow the device to settle. + input_mapping: + wait_time: {type: direct, value: 2} + output_mapping: {} + - steptype: PythonModuleStep + step_name: Take measurement + description: Read and validate the configured voltage. + action_type: method + module: device_driver.py + method_name: read_measurement + input_mapping: {} + output_mapping: + measured_value: {type: local, local_name: measurement} + status: {type: range, min: 4.8, max: 5.2} +teardown_steps: + - steptype: PythonModuleStep + step_name: Disconnect device + description: Close the device connection during teardown. + action_type: method + module: device_driver.py + method_name: disconnect + input_mapping: {} + output_mapping: {} diff --git a/docs/source/_examples/instrument_recipe_v2.yml b/docs/source/_examples/instrument_recipe_v2.yml new file mode 100644 index 0000000..6d9a583 --- /dev/null +++ b/docs/source/_examples/instrument_recipe_v2.yml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: CC-BY-SA-4.0 +--- +name: Frequency stability test +version: "1.0" +recipe_version: 2.0.0 +description: Read a CNT-91 through a packaged Python test method. +main_sequence: Main +test_package: my_project.tests +globals: + device_name: USB0::0x14EB::0x0091::205575::INSTR +--- +sequence_name: Main +description: Run the frequency measurement. +parameters: {} +locals: {} +outputs: {} +setup_steps: [] +steps: + - steptype: PythonModuleStep + step_name: Read CNT-91 + description: Connect to the counter and evaluate its measurement. + action_type: method + module: instrument_tests.py + method_name: run_cnt91 + input_mapping: + device_name: {type: global, global_name: device_name} + output_mapping: + output: {type: passfail} +teardown_steps: [] diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index e851ff8..cacdc19 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -156,24 +156,15 @@ The ``PythonModuleStep`` now uses a completely rewritten ``__load_module`` metho * Removed module conflict detection (handled by Python's import system) * Streamlined error handling with proper exception chaining -**Recipe Configuration Example**: +**Recipe configuration fields**: .. code-block:: yaml - --- - name: FSI PTS - version: 0.0.1 - description: Test recipe for FSI version check - test_package: fsi_pts.tests # NEW: Specifies package containing test modules - globals: {} - - --- - sequence_name: Main - steps: - - steptype: PythonModuleStep - step_name: Get FSI Status - module: test_status.py # Resolved to fsi_pts.tests.test_status - method_name: test_status + # RecipeHeader fragment + test_package: fsi_pts.tests + + # PythonModuleStep fragment + module: test_status.py # Resolves to fsi_pts.tests.test_status **Benefits**: @@ -310,21 +301,9 @@ For migrating to resource-based module loading: .. code-block:: yaml - # Old (file-based) - --- - name: My Recipe - globals: {} - - steps: - - steptype: PythonModuleStep - module: tests/my_test.py # File path - - # New (resource-based) - --- - name: My Recipe - test_package: my_package.tests # Package containing test modules - globals: {} - - steps: - - steptype: PythonModuleStep - module: my_test.py # Resolved to my_package.tests.my_test + # File-based module field + module: tests/my_test.py + + # Resource-based header and module fields + test_package: my_package.tests + module: my_test.py # Resolves to my_package.tests.my_test diff --git a/docs/source/instruments.rst b/docs/source/instruments.rst index fcee15f..282b34d 100644 --- a/docs/source/instruments.rst +++ b/docs/source/instruments.rst @@ -277,29 +277,9 @@ Instrument functions are called from recipe YAML steps via the ``PythonModuleStep`` mechanism (see :ref:`yaml_format`). The function receives its arguments from the recipe's variable scope. -.. code-block:: yaml - :caption: Example recipe step calling the CNT-91 - - --- - name: Frequency Stability Test - version: "1.0" - recipe_version: "1.1.0" - test_package: pypts.Instrument_test_examples - globals: - device_name: "USB0::0x14EB::0x0091::205575::INSTR" - gate_time: 0.01 - n_samples: 500 - - --- - name: Main - steps: - - type: python_module - function: run_cnt91 - args: - device_name: ${device_name} - gate_times: ${gate_time} - samples: ${n_samples} - channels: 1 +.. literalinclude:: _examples/instrument_recipe_v2.yml + :language: yaml + :caption: Complete recipe calling the CNT-91 The function must follow the pypts step contract: it must return a dict with at least the key ``"output"`` set to ``True`` on success. diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index af8488c..61081e8 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -8,8 +8,8 @@ Recipe Language 2 Architecture .. important:: Recipe language ``2.0.0`` is the only production parsing and execution - path. Bundled version 1 recipes remain intentionally unmigrated and are - rejected with migration diagnostics. + path. Maintained recipes and setup templates use version 2; external + version 1 files are rejected with migration diagnostics. Exact version 2 fields, types, defaults, and examples are in the generated :doc:`_generated/recipe_language_reference`. The aggregate schema is also available as @@ -90,8 +90,9 @@ semantics therefore remain separate stages:: ``parse_recipe_text`` and ``parse_recipe_file`` return ``ParseResult``. A valid result owns an aggregate :ref:`recipe-v2-header` plus one or more :ref:`recipe-v2-sequence` models. ``require_recipe()`` raises with the complete -diagnostic tuple when errors exist. ``dump_recipe`` writes canonical version 2 -YAML; comments and original formatting are not a round-trip guarantee. +diagnostic tuple when errors exist. ``recipe_to_yaml`` returns deterministic +version 2 YAML without file I/O; comments, quoting, and original formatting +are not preserved. Parse/serialize/reparse preserves the aggregate definition. Here, "composition" is PyYAML terminology, not a PyPTS adapter or an additional recipe representation. ``yaml.compose_all(..., Loader=yaml.SafeLoader)`` diff --git a/docs/source/report_generation.rst b/docs/source/report_generation.rst index ba3bac0..b0d9440 100644 --- a/docs/source/report_generation.rst +++ b/docs/source/report_generation.rst @@ -117,6 +117,7 @@ Test methods executed by ``PythonModuleStep`` can return image file paths (PNG, - steptype: PythonModuleStep step_name: Analyse Signal + description: Analyse the signal and publish its plot. module: my_tests.py action_type: method method_name: run_signal_analysis diff --git a/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst index 81a7d4b..d2e9e74 100644 --- a/docs/source/troubleshooting.rst +++ b/docs/source/troubleshooting.rst @@ -56,75 +56,20 @@ Before using pypts, you may need to install the following system dependencies fo Recipe-related issues. ----------------------------------------- -Issues related to the recipe are often related to a difference or lack of keys. - -**Required framework for recipe** - -The specifics in the framework below is required in the prelude of the recipe to run the framework. - -.. code-block:: yaml - - name: Example Test Recipe - version: 0.1.0 - recipe_version: 1.0.0 - description: A sample description of a recipe - main_sequence: Main - test_package: test_package - globals: {} - -The Main sequence is also required and consists of the rest of the test cases which exists of the following elements. - -.. code-block:: yaml - - sequence_name: Main - description: The main sequence of steps for the example recipe. - parameters: - target_value: '0' - locals: - target_value: '45' - test_name: Hello - outputs: - my_output: None - setup_steps: [] - steps: - - steptype: UserInteractionStep - step_name: Are you all right? - description: Asking user for something - skip: false - input_mapping: - message: - type: direct - value: 'example' - image_path: - type: direct - value: example.jpg - options: - type: direct - value: - - 'yes': '' - - 'no': '' - output_mapping: - user_response: - type: equals - value: 'yes' - - steptype: PythonModuleStep - step_name: Run a other_test - action_type: method - module: example_tests.py - method_name: other_test - input_mapping: {} - output_mapping: - some_return: - type: passfail - value: - type: local - local_name: test_value - -Above we see an example of an UserInteractionStep type and a PythonModuleStep setup. The UserInteractionStep is used for when the system is awaiting an action from user. -The PythonModuleStep shows a requirement for determining which module to use and a specification of the method_name to be used. - -.. note:: - Notice that the output_mapping for ``UserInteractionStep`` is user_response, respective to a response on a pushed button. +Recipe language ``2.0.0`` requires a header document and at least one complete +sequence document. Start from the parser-tested example instead of copying an +isolated fragment: + +.. literalinclude:: _examples/recipe_v2.yml + :language: yaml + :caption: Valid recipe-language 2 structure + +If validation fails, use the diagnostic code, field path, and source position. +The most common migration failures are an old or missing ``recipe_version``, a +lowercase step discriminator, a literal input without ``type: direct``, a +missing description, or the removed sequence ``serial_number`` field. See +:ref:`yaml_format` for migration guidance and the generated +:doc:`_generated/recipe_language_reference` for exact fields. **ModuleNotFoundError** diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 149fae5..5d6dd17 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -195,62 +195,10 @@ The pyproject file operates similarily to a makefile and is the construction of Create a YAML file defining your test sequence. The recipe consists of a main document defining metadata and global variables, followed by documents defining named sequences. See :ref:`yaml_format` for full explaination of all steps available for recipe. -.. code-block:: yaml +.. literalinclude:: _examples/device_recipe_v2.yml + :language: yaml :caption: my_recipe.yaml - # Main recipe definition - name: MyTestRecipe - description: Example recipe demonstrating basic steps. - version: "1.0" - globals: - device_address: "COM3" - test_voltage: 5.0 - --- - # Main sequence definition - sequence_name: Main - parameters: - # Input parameters for this sequence (if any) - locals: - # Local variables initialized for this sequence - measurement: null - outputs: - # Values from locals to expose as sequence output - - measurement - setup_steps: [] - steps: - - steptype: WaitStep - step_name: Initial Delay - input_mapping: - wait_time: { type: direct, value: 2 } - - steptype: PythonModuleStep - step_name: Configure Device - module: device_driver.py - action_type: method - method_name: setup_device - input_mapping: - port: { type: global, global_name: device_address } - voltage: { type: global, global_name: test_voltage } - output_mapping: - success: { type: passfail } - - steptype: PythonModuleStep - step_name: Take Measurement - module: device_driver.py - action_type: method - method_name: read_measurement - input_mapping: {} - output_mapping: - measured_value: { type: local, local_name: measurement } - # Example pass/fail check - status: { type: range, min: 4.8, max: 5.2 } - teardown_steps: - - steptype: PythonModuleStep - step_name: Disconnect Device - module: device_driver.py - action_type: method - method_name: disconnect - input_mapping: {} - output_mapping: {} - .. note:: Replace ``device_driver.py`` with the actual name of your Python module containing the methods called by ``PythonModuleStep``. Adding the path should not be done as the system automatically detects the path to the specified test. Therefore, avoid naming modules the same unless you specify a ``test_package`` as described below. @@ -261,54 +209,16 @@ Alternative: Resource-Based Module Loading For better distribution and deployment, you can use resource-based module loading by organizing your test modules as Python packages: +The complete example above already uses resource-based loading. The focused +fields are: + .. code-block:: yaml - :caption: my_recipe.yaml (resource-based) - - # Main recipe definition with test_package - name: MyTestRecipe - description: Example recipe using resource-based loading. - version: "1.0" - test_package: my_project.tests # NEW: Package containing test modules - globals: - device_address: "COM3" - test_voltage: 5.0 - --- - # Main sequence definition - sequence_name: Main - parameters: {} - locals: - measurement: null - outputs: - measurement: {} - setup_steps: [] - steps: - - steptype: PythonModuleStep - step_name: Configure Device - module: device_driver.py # Resolved to my_project.tests.device_driver - action_type: method - method_name: setup_device - input_mapping: - port: { type: global, global_name: device_address } - voltage: { type: global, global_name: test_voltage } - output_mapping: - success: { type: passfail } - - steptype: PythonModuleStep - step_name: Take Measurement - module: device_driver.py # Same module, different method - action_type: method - method_name: read_measurement - input_mapping: {} - output_mapping: - measured_value: { type: local, local_name: measurement } - status: { type: range, min: 4.8, max: 5.2 } - teardown_steps: - - steptype: PythonModuleStep - step_name: Disconnect Device - module: device_driver.py - action_type: method - method_name: disconnect - input_mapping: {} - output_mapping: {} + + # Header fragment + test_package: my_project.tests + + # PythonModuleStep fragment + module: device_driver.py # Resolves to my_project.tests.device_driver **Package Structure Example**: @@ -421,26 +331,12 @@ Add the dotted ``test_package`` field and make module paths relative to it: .. code-block:: yaml - # Before - --- - name: My Recipe - globals: {} - - steps: - - steptype: PythonModuleStep - module: tests/test_module1.py # File path - method_name: my_test - - # After - --- - name: My Recipe - test_package: my_project.tests # NEW: Package specification - globals: {} - - steps: - - steptype: PythonModuleStep - module: test_module1.py # Relative to my_project.tests - method_name: my_test + # File-based module field + module: tests/test_module1.py + + # Resource-based header and module fields + test_package: my_project.tests + module: test_module1.py # Relative to my_project.tests **Step 3: Install and Test** diff --git a/docs/source/yaml_format.rst b/docs/source/yaml_format.rst index 5e6af14..dca8d42 100644 --- a/docs/source/yaml_format.rst +++ b/docs/source/yaml_format.rst @@ -1,1096 +1,128 @@ -.. SPDX-FileCopyrightText: 2025 CERN +.. SPDX-FileCopyrightText: 2026 CERN .. .. SPDX-License-Identifier: CC-BY-SA-4.0 -.. important:: - - Production parsing and execution require recipe language ``2.0.0``. See - :doc:`recipe_language_architecture` for its flow and - :doc:`_generated/recipe_language_reference` for the generated syntax - reference. Bundled version 1 examples remain unmigrated and are rejected. - .. _yaml_format: -#################### Recipe YAML Format -#################### - -The recipe file is a multi-document YAML file. The first document defines -the main recipe metadata and global variables, while subsequent documents -define individual sequences that make up the test flow. - -Document 1: Main Recipe Configuration -====================================== - -.. code-block:: yaml - :caption: Example Main Recipe Document - - --- - name: Name of the recipe. Typically the project name. - version: Allows for tracking different versions of the file - recipe_version: Optional version of the recipe format specification. - description: A more complete description of this recipe - main_sequence: Main # Optional: Name of the sequence to run by default. Defaults typically to "Main". - test_package: my_package.tests # Optional: Python package containing test modules for PythonModuleStep - continue_on_error: false # Optional policy overriding every step's continue_on_error value. - globals: # Globals can be referenced and used from any step in the whole file - global_name: value - other_global: other_value - # ... - # tags: # Optional tags (Currently commented out in code) - # key1: value1 - -Main Recipe Configuration Fields -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -* **name** (str): Name of the recipe, typically the project name. -* **version** (str): Version string for tracking different versions of the recipe. -* **recipe_version** (str, optional): Version of the recipe format specification. Error-policy behavior is not gated by this field. -* **description** (str): A detailed description of the recipe's purpose. -* **main_sequence** (str, optional): Name of an existing sequence to run by default. If omitted, ``Main`` is selected and a sequence with that name must exist. -* **report** (str, optional): Selects whether new test results should overwrite the previous report (``overwrite``) or should be added to the report file (``append``). Defaults to ``overwrite``. -* **test_package** (str, optional): Importable Python package containing test modules for ``PythonModuleStep``. Dotted names such as ``my_project.tests`` are supported. When specified, modules are imported by package name without filesystem discovery. See :ref:`resource_based_loading`. -* **continue_on_error** (bool, optional): Recipe-wide policy that overrides every step-level value. It is not a global variable. -* **globals** (dict): Global variables that can be referenced from any step in the recipe. - -``continue_on_error`` is valid as a recipe-level field or a step field. When -present in the recipe header, its boolean value overrides every step-level value. - -.. _resource_based_loading: - -Resource-Based Module Loading -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When ``test_package`` is specified, ``PythonModuleStep`` imports test modules from that package instead of discovering them from filesystem paths: - -**Benefits:** - * Modules are bundled with your package during distribution - * No dependency on current working directory - * Uses Python's standard import mechanism - * More reliable deployment - -**Example:** - -.. code-block:: yaml - - --- - name: My Recipe - test_package: my_project.tests - globals: {} - - --- - sequence_name: Main - steps: - - steptype: PythonModuleStep - module: test_module.py # Resolves to my_project.tests.test_module - action_type: method - method_name: my_test - -.. note:: - Notice the indentation inside ``steps`` and the ``-`` in front of the step. Adding this - is crucial for the functionality of the recipe. - - -**Package Structure Required:** - -.. code-block:: text - - my_project/ - ├── __init__.py - ├── tests/ - │ ├── __init__.py # Required for Python package - │ ├── test_module.py - │ └── other_tests.py - └── recipe.yaml - -**Migration from File-Based:** - * Add ``__init__.py`` files to make directories into packages - * Add ``test_package`` field to recipe - * Use module paths relative to ``test_package``; nested paths are supported - * Install your package with ``pip install -e .`` - ---- # Separator for the next document - -Document 2...N: Sequence Definition -=================================== - -When an output mapping contains multiple ``passfail``, ``equals``, or ``range`` -checks, all checks must pass for the step to be ``PASS``. A failed check makes -the step ``FAIL`` regardless of mapping order. ``passthrough`` represents an -already-computed result and must be the only verdict-producing mapping (it may -still be accompanied by metadata outputs such as ``local`` or ``global``). - -For example, this step passes only when both checks pass; reversing their YAML -order does not change the result: - -.. code-block:: yaml - - output_mapping: - powered: {type: passfail} - voltage: {type: range, min: 4.8, max: 5.2} - -Each subsequent document defines a sequence. - -.. code-block:: yaml - :caption: Example Sequence Document - - sequence_name: Name of the sequence. A sequence defines a list of steps - description: Description of the sequence - setup_steps: [] # Steps that run first and are used to setup environments. Typically utility steps necessary for the next ones to work properly. - steps: [] # Main steps of the sequence. These are run in order by the execution environment - teardown_steps: [] # Teardown steps are run even if there is an error during the run. This is to make sure we run some shutdown routines no matter what happens. - locals: # List of variables local to the sequence in scope (contrasted with global variables defined in recipe document) - local_name: local_value - # ... - parameters: {} # Parameter metadata, keyed by parameter name - outputs: {} # Output metadata, keyed by output name - -``parameters`` and ``outputs`` are dictionaries. They are currently descriptive -metadata; the runtime does not yet bind subsequence outputs automatically. +================== +PyPTS accepts recipe language ``2.0.0``. Exact fields, types, required status, +defaults, aliases, and discriminator values are generated from the production +models in the :doc:`_generated/recipe_language_reference`. The downloadable +:download:`JSON Schema <_generated/recipe_language.schema.json>` describes the +same contract. -.. _step_definition_details: +A complete maintained example is included below. This file is parsed by the +test suite with the production parser. -Step Definition -=============== +.. literalinclude:: _examples/recipe_v2.yml + :language: yaml + :caption: Complete recipe-language 2 example -Each item in `setup_steps`, `steps`, and `teardown_steps` is a dictionary representing a Step. +Multi-document structure +------------------------ -.. code-block:: yaml - :caption: Example Step Structure - - steptype: PythonModuleStep | UserInteractionStep | SequenceStep | WaitStep | ... # Determines the type of action - step_name: Name of the step # A descriptive name for the step - id: unique_id # Optional: A unique identifier. Defaults to a generated UUID. - description: More details about the step # Optional: More details about the step's purpose. - skip: false # Optional: If true, the step execution is skipped. Defaults to false. - critical: false # Optional: If true, errors in this step always stop execution. Defaults to false. - # --- Fields specific to certain steptypes --- - action_type: method # e.g., For PythonModuleStep: 'method', 'read_attribute', 'write_attribute' - module: path/to/my_module.py # e.g., For PythonModuleStep: Path to the Python file - method_name: my_function # e.g., For PythonModuleStep with action_type 'method': Name of the function to run - # ... other specific fields depending on steptype ... - # --- Input/Output Mapping --- - input_mapping: {} # Defines how the step gets its input data. See below. - output_mapping: {} # Defines how the step's output is processed. See below. - continue_on_error: False - - -Key fields common to most steps: - -* ``steptype`` (str): Determines the type of action (e.g., ``PythonModuleStep``, ``SequenceStep``, ``UserInteractionStep``, ``WaitStep``). Specific step types may have additional required or optional fields. -* ``step_name`` (str): A descriptive name for the step. -* ``id`` (str, optional): A unique identifier. Defaults to a generated UUID. -* ``description`` (str, optional): More details about the step's purpose. -* ``skip`` (bool, optional): If ``true``, the step execution is skipped. Defaults to ``false``. -* ``critical`` (bool, optional): If ``true``, errors in this step always stop execution, even when ``continue_on_error`` is enabled. Defaults to ``false``. -* ``input_mapping`` (dict): Defines how the step gets its input data. See :ref:`input_mapping_details`. -* ``output_mapping`` (dict): Defines how the step's output is processed and stored. See :ref:`output_mapping_details`. -* ``continue_on_error`` (bool, optional): Continue after this step produces ``ERROR``. Defaults to ``false`` and is overridden by ``globals.continue_on_error`` when that global exists. It does not alter ``FAIL`` behavior. +A recipe is one YAML stream containing multiple documents, separated by +``---``: +* The first document is a :ref:`recipe-v2-header`. Its required + ``recipe_version`` is ``2.0.0`` and ``main_sequence`` names an existing + sequence. +* Every later document is a :ref:`recipe-v2-sequence`. Sequence names must be + unique, and at least one sequence is required. +* Each sequence owns ``setup_steps``, ``steps``, and ``teardown_steps`` lists. + Step definitions use one of the exact, case-sensitive canonical + discriminators listed in the generated reference. -.. _input_mapping_details: +Validation is strict. Unknown fields, wrong scalar types, duplicate YAML keys, +unsafe YAML tags, missing required fields, invalid sequence references, and +unsupported versions produce diagnostics. PyPTS does not repair or silently +normalize legacy syntax. +Variables and mappings +---------------------- -Input Mapping Details (``input_mapping``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``globals`` belong to the recipe header and are available throughout the run. +``locals`` belong to one active sequence. ``parameters`` and ``outputs`` are +currently reserved metadata dictionaries; the runtime does not automatically +bind nested-sequence inputs or outputs from them. -The ``input_mapping`` dictionary maps internal step input names (e.g., argument names for a ``PythonModuleStep``) to data sources. The keys of ``input_mapping`` are the names the step internally uses for its inputs, and the values specify where that data comes from. - -Each value in the ``input_mapping`` dictionary is *another* dictionary with the following keys: - -* ``type`` (str, optional): Source of the value. Must be one of ``direct``, ``local``, or ``global``. If omitted, defaults to ``direct``. -* ``value``: Required if ``type`` is ``direct`` (or omitted). Provides the literal value directly. -* ``local_name``: Required if ``type`` is ``local``. Specifies the name of the sequence's local variable to read from. -* ``global_name``: Required if ``type`` is ``global``. Specifies the name of the recipe's global variable to read from. -* ``indexed`` (bool, optional): Defaults to ``false``. If ``true`` for one or more inputs, the step becomes an ``IndexedStep`` internally. It runs multiple times, once for each item in the *shortest* input list marked as ``indexed: true``. Non-indexed inputs are repeated (their value is used as-is) for each run of the indexed step. +Every input mapping has an explicit ``type``: .. code-block:: yaml - :caption: Example Input Mapping Options input_mapping: - # Input 'arg1' gets the literal integer value 3 (type defaults to direct) - arg1: {value: 3, indexed: false} - - # Input 'arg2' gets its value from the sequence's local variable 'my_local_var' - arg2: {type: local, local_name: my_local_var, indexed: false} - - # Input 'arg3' gets its value from the recipe's global variable 'my_global_var' - arg3: {type: global, global_name: my_global_var, indexed: false} - - # Input 'items_to_process' comes from a direct list. - # Because indexed is true, the step will run 3 times. - # Run 1: items_to_process = 1 - # Run 2: items_to_process = 2 - # Run 3: items_to_process = 3 - # Inputs arg1, arg2, arg3 will keep their mapped values for each of these runs. - items_to_process: {type: direct, value: [1, 2, 3], indexed: true} - - -.. _indexed_step_naming: - -Indexed Step Naming -"""""""""""""""""""" - -By default, each iteration of an indexed step is named -``" [1/N]"``, ``" [2/N]"``, etc. To produce more -informative report lines, use **Python format placeholders** in ``step_name`` -that reference the indexed input names. The framework substitutes the actual -value for each iteration automatically. - -.. code-block:: yaml - - - steptype: PythonModuleStep - step_name: "ADC-03 axis {axis}" - module: tests/adc.py - action_type: method - method_name: test_adc_03_axis - input_mapping: - axis: {type: direct, value: [0, 1, 2, 3, 4, 5, 6, 7], indexed: true} - output_mapping: - passed: {type: passfail} - -This produces eight report lines: ``ADC-03 axis 0``, ``ADC-03 axis 1``, ..., -``ADC-03 axis 7``. - -Standard Python format specifications are supported — for example -``"AO-02 CH{channel:02d} FFT"`` produces ``AO-02 CH00 FFT``, -``AO-02 CH01 FFT``, etc. - -If the ``step_name`` does not contain any ``{...}`` placeholders, the -default ``[i/N]`` suffix is used. - - -.. _output_mapping_details: - -Output Mapping Details (``output_mapping``) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The ``output_mapping`` dictionary defines how the step's raw output is processed, evaluated for pass/fail status, and stored back into variables. The keys of the ``output_mapping`` dictionary correspond to the keys in the step's raw output data (typically a dictionary). If the step produces a non-dictionary output (e.g., a ``PythonModuleStep`` method returns a single value like a boolean or number), it's treated as a dictionary with a single key ``output`` (e.g., ``{"output": returned_value}``). - -Each value in the ``output_mapping`` dictionary is *another* dictionary specifying the action to take: - -* ``type`` (str): How to handle the output value associated with this key. Must be one of ``local``, ``global``, ``passfail``, ``equals``, ``range``, ``passthrough``, or ``image``. -* ``local_name``: Required if ``type`` is ``local``. The name of the sequence's local variable where this output value should be stored. -* ``global_name``: Required if ``type`` is ``global``. The name of the recipe's global variable where this output value should be stored. -* ``value``: Required if ``type`` is ``equals``. The target value for comparison. If the step's output value for this key equals ``value``, the check passes. -* ``min``, ``max``: Required if ``type`` is ``range``. The inclusive lower (``min``) and upper (``max``) bounds for comparison. If the step's output value for this key falls within [min, max], the check passes. -* ``passthrough``: Used to propagate a `ResultType` directly. This is often used with the implicit ``__result`` output key from a `SequenceStep` to propagate the overall status of the subsequence, or internally by `IndexedStep` to represent the aggregate result. -* ``image``: The output value is treated as a file path to an image (PNG, JPG, SVG, etc.). The framework copies the file into ``/img/`` and embeds it in the HTML report at the bottom of the page, captioned with the step name and result. Does not affect pass/fail determination. - -**Pass/Fail Determination:** - -* If any output key is mapped with ``type: passfail``, the boolean value of that output determines the step's Pass/Fail status. -* If any output key is mapped with ``type: equals`` or ``type: range``, the comparison result determines the step's Pass/Fail status. If multiple such mappings exist, *all* must pass for the step to pass. -* If *no* output keys are mapped to ``passfail``, ``equals``, or ``range``, the step automatically finishes with a status of ``DONE``, which is generally treated as equivalent to ``PASS``. - -.. code-block:: yaml - :caption: Example Output Mapping Options - - output_mapping: - # Store the value associated with the output key 'result_data' - # into the local variable 'my_local_result'. - result_data: {type: local, local_name: my_local_result} - - # Store the value associated with the output key 'shared_value' - # into the global variable 'my_global_result'. - shared_value: {type: global, global_name: my_global_result} - - # Use the boolean value associated with the output key 'test_passed' - # to determine if the step passes or fails. - test_passed: {type: passfail} - - # Check if the value associated with the output key 'status_code' - # is exactly equal to 200. If yes, pass; otherwise, fail. - status_code: {type: equals, value: 200} - - # Check if the value associated with the output key 'measurement' - # is between 3.0 and 6.5 (inclusive). If yes, pass; otherwise, fail. - measurement: {type: range, min: 3.0, max: 6.5} - - # For SequenceStep: Propagate the overall Pass/Fail/Done status - # of the subsequence using its implicit '__result' output. - __result: { type: passthrough } - - # Copy the image file at the returned path into the report directory - # and embed it in the HTML report with the step name and result as caption. - chart_image: { type: image } - - -.. _data_collection_vs_assertion: - -Separating Data Collection from Assertion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The output mapping system supports a powerful pattern: **decoupling data -collection from pass/fail assertion**. This is achieved by using ``global`` -or ``local`` output types, which store raw data without performing any -assertion. When *only* storage types are used in a step's output mapping, -the step finishes with ``ResultType.DONE`` (neutral) — it neither passes -nor fails. - -**Why use this pattern?** - -* Keep test methods focused on *acquiring* data, not *judging* it. -* Reuse the same acquisition step with different assertion criteria. -* Inspect or log raw data before deciding pass/fail in a later step. -* Run multiple validations on the same dataset without re-acquiring. - -**Example — collect first, assert later:** - -.. code-block:: yaml - - # Step 1: Pure data collection — no assertion - - steptype: PythonModuleStep - step_name: Acquire ADC channels - module: tests/adc.py - action_type: method - method_name: read_all_channels - input_mapping: - ssh_client: {type: global, global_name: ssh_client} - output_mapping: - channels: - type: global - global_name: adc_raw_data # Raw dict stored, step returns DONE - - # Step 2: Assert on the stored data - - steptype: PythonModuleStep - step_name: Validate ADC channels - module: tests/adc.py - action_type: method - method_name: validate_channels - input_mapping: - data: {type: global, global_name: adc_raw_data} - output_mapping: - passed: {type: passfail} # Boolean assertion - -The Python method in step 1 only needs to return the raw data: - -.. code-block:: python - - def read_all_channels(ssh_client=None, **kwargs): - # ... acquire data ... - return {"channels": channel_dict} # No pass/fail logic here - -And the validation method in step 2 receives the stored data and judges it: - -.. code-block:: python - - def validate_channels(data=None, **kwargs): - all_ok = all(abs(v - expected) < tolerance for v, expected in ...) - return {"passed": all_ok} - -**Example — mixed: assert AND store in one step:** - -A single step can combine assertion types with storage types. This is useful -when you want an immediate pass/fail verdict *and* want to keep the raw data -available for later steps or reporting: - -.. code-block:: yaml - - output_mapping: - passed: - type: passfail # Determines step PASS/FAIL - raw_readings: - type: global - global_name: saved_readings # Also stored for later use - summary: - type: local - local_name: test_summary # Stored in sequence-local scope - -**Summary of assertion vs. storage types:** - -.. list-table:: - :header-rows: 1 - - * - Type - - Behaviour - - Determines pass/fail? - * - ``passfail`` - - Boolean → PASS / FAIL - - Yes - * - ``equals`` - - Compare to target value - - Yes - * - ``range`` - - Check within [min, max] - - Yes - * - ``passthrough`` - - Propagate a ResultType - - Yes - * - ``global`` - - Store in global variable - - No — step returns DONE - * - ``local`` - - Store in local variable - - No — step returns DONE - * - ``image`` - - Copy image file into report and embed in HTML - - No — step returns DONE - -.. note:: - If a step's output mapping contains **only** ``global`` and/or ``local`` - entries (no assertion types), the step always finishes with - ``ResultType.DONE``. This is treated as a neutral/successful completion — - it will not trigger ``continue_on_error`` or ``critical`` stop behaviour. - - -Step formatting -=================== - -There are multiple ways of formatting the recipe YAML file. -One way of doing it is compressing the arguments of ``input_mapping`` and ``output_mapping``. An example is visible below. - -.. code-block:: yaml - - steptype: PythonModuleStep - step_name: Call Python Function - description: Describe the action of the step - module: my_module.py - action_type: method - method_name: my_function - input_mapping: - arg1: { type: direct, value: "hello" } - arg2: { type: local, local_name: local_var1 } - output_mapping: - result: { type: local, local_name: output_data } - passed: { type: passfail } # Treats boolean output as pass/fail - -The other way of formatting each step is to expand the outputs so we get - -.. code-block:: yaml - - steptype: PythonModuleStep - step_name: Call Python Function - description: Describe the action of the step - module: my_module.py - action_type: method - method_name: my_function - input_mapping: - arg1: - type: direct - value: "hello" - arg2: - type: local - local_name: local_var1 - output_mapping: - result: - type: local - local_name: output_data - passed: - type: passfail # Treats boolean output as pass/fail - -In this formatting, the indentation of input elements like ``type`` and ``value`` are important to achieve similar structure. The indentation scheme is of same structure as python programming. -Both work as intendedn and either can be used, even mixed together. - -Specific Step Types -=================== - -Given the required different functionalities required for testing, multiple step types were developed. This section descrives the types, functionality, format and what input/output. The following are the available steptypes: - - PythonModuleStep - - WaitStep - - UserInteractionStep - - SSHConnectStep - - SSHUploadStep - - UserLoadingStep - - UserRunMethodStep - - UserWriteStep - - SerialNumberStep - -Each has its own template of required elements but with overlapping types of elements. -The elements ``step_name`` and ``description`` are not explained further in this section as they're descriptive elements for reporting and GUI with no change between steps. - -.. note:: - The optional arguments ``critical``, ``skip`` and ``continue_on_error`` all apply to the following step types. Check :ref:`step_definition_details` for information on ``skip`` and ``critical`` and :ref:`continue_on_error_details` for information on ``continue_on_error``. - - - -PythonModuleStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Executes Python method or code. - -.. code-block:: yaml - - steptype: PythonModuleStep - step_name: Call Python Function #the name of the - description: Describe the action of the step - module: my_module.py #the name of the file with the test initialization - action_type: method # Or 'read_attribute', 'write_attribute' - method_name: my_function # Required if action_type is 'method' - input_mapping: - arg1: { type: direct, value: "hello" } - arg2: { type: local, local_name: local_var1 } - # For read_attribute: - # attribute_name: { type: direct, value: "my_attr" } - # For write_attribute: - # attribute_name: { type: direct, value: "my_attr" } - # attribute_value: { type: direct, value: 10 } - output_mapping: - result: { type: local, local_name: output_data } - passed: { type: passfail } # Treats boolean output as pass/fail - -* ``module`` (str): Python module path. With ``test_package``, it is relative to that package (for example ``test_module.py`` or ``helpers/test_module.py``). -* ``action_type`` (str): ``method``, ``read_attribute``, or ``write_attribute``. -* ``method_name`` (str): Name of the method to call. -* ``input_mapping`` (dictionary): Inputs to the method. Each input(``arg``) is required to have a ``type``. -* ``output_mapping`` (dictionary): outputs of the method. Each output(``arg``) is required to have a ``type``. If multiple outputs, ensure the returned output of method has same label, example ``passed`` as given in output_mapping. - - -WaitStep -~~~~~~~~~~~~~~~~~~~~ - -Waits the specified period of time in seconds before next step starts. - -.. code-block:: yaml - - steptype: WaitStep - step_name: Wait for 3s - description: Waiting 3 seconds on this step - skip: false - input_mapping: - wait_time: - value: '3' - -* ``steptype`` (str): Determines the type of action. -* ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. - - -UserInteractionStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Allows for user to interact with gui through action on buttons. It allows for adding many buttons but it requires at least one button if test is to be successful. - -.. code-block:: yaml - - - steptype: UserInteractionStep - step_name: Start test - input_mapping: - message: {type: direct, value: "Connect the WRS as shown and click Yes", indexed: false} - image_path: {type: direct, value: "test.png"} - options: {type: direct, value: [{'yes': 'yes'},{two: 'no'}], indexed: false} - output_mapping: - output: {type: equals, value: "yes"} - -* ``steptype`` (str): Determines the type of action. -* ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. -* ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. - -These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. - -* ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. -* ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. - - - -SSHConnectStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Allows to setup a SSH connection to be used globally during the test. - -.. code-block:: yaml - - steptype: SSHConnectStep - step_name: Find the SSH client connection - description: Asking user to find the file needed - continue_on_error: false - -* ``steptype`` (str): Determines the type of action. - -This steptype has certain **requirements** for which globals exist. The required globals are: - -* ``ssh_client: None`` : The variable holding the opened paramiko client to be called in functions. -* ``host: 129`` :The SSH hostname or IP address. -* ``user: username``:The SSH username -* ``password: None`` :Password for SSH auth. -* ``private_key: 'path/to/your/key_file'`` :The path to key file. important if no password is given. -* ``port: None`` (int): SSH port (default: 22). - -If password is not supplied, the function will automatically use ``private_key`` as verification. - -**Important**. When this steptype is used, **always** put steptype ``SSHCloseStep`` under ``teardownsteps``. It can be placed multiple times, but always one in ``teardownsteps``. - -.. code-block:: yaml - - teardown_steps: - - steptype: SSHCloseStep - step_name: Closes the SSH client - -To use the SSH client, add ``ssh_client`` to input_mapping. - -.. code-block:: yaml - - target: - type: global - global_name: ssh_client - -An Example of using it for a function can be seen below. - -.. code-block:: python - - def write_a_simple_filessh(target): - target.exec_command("echo 'Hello World' > myfile.txt") - - return (True) - - - -SSHUploadStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Uploads one or more local files to the remote host via SFTP. Uses the ``ssh_client`` global -set by ``SSHConnectStep``. Compares local and remote SHA-256 digests before uploading, so -repeated recipe runs skip files that are already up to date. - -.. code-block:: yaml - - - steptype: SSHUploadStep - step_name: Deploy C tool binaries - description: Upload ARM ELF binaries to /tmp/ on the test target - local_package: my_pts_package - files: - - local: bin/tool-a - remote: /tmp/tool-a - - local: bin/tool-b - remote: /tmp/tool-b - permissions: "0o755" - skip_if_sha256_match: true - continue_on_error: false - output_mapping: - passed: - type: passfail - -* ``files`` (list, required): List of ``{local: ..., remote: ...}`` pairs. - ``local`` is a path string resolved relative to the project root, or relative - to ``local_package`` when that field is set. -* ``local_package`` (str, optional): Name of an installed Python package. - When set, ``local`` paths are resolved via ``importlib.resources`` inside - that package (use this for packaged binaries or data files). -* ``permissions`` (int or str, optional): ``chmod`` value applied after each - upload. Accepts an integer (e.g. ``493``) or an octal string (e.g. ``"0o755"``). - Default: ``0o755`` (rwxr-xr-x). -* ``skip_if_sha256_match`` (bool, optional): Skip the upload when the local - and remote SHA-256 digests match. Default: ``true``. -* ``continue_on_error`` (bool, optional): If ``true``, an upload failure does - not abort the recipe. Default: ``false``. - -The step returns ``{"passed": bool, "deployed": [names], "skipped": [names]}``. -Map ``passed`` to ``type: passfail`` in ``output_mapping`` for automatic -PASS/FAIL verdict reporting. - -**Important**: ``SSHConnectStep`` must appear before ``SSHUploadStep`` in -``setup_steps`` so that the ``ssh_client`` global is populated. - -UserLoadingStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Used to load a file to be used somewhere else. Could fx be a calibration file or configuration file for certain instruments. - -.. code-block:: yaml - - steptype: UserLoadingStep - step_name: Load config file - description: Asking user to find the file needed - input_mapping: - message: - type: direct - value: 'Find the specified file' - image_path: - type: direct - value: lego2.jpg - options: - type: direct - value: - - cancel: 'cancel' - - file: 'next' - output_mapping: - output: - type: passfail - file_save_location: - type: global - variable: file - -* ``steptype`` (str): Determines the type of action. -* ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. -* ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. - -These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. - -* ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. -* ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. -* ``file_save_location`` (dict, optional): Variable to save the loaded file. . If not existing, will default to ``type: local, variable: file``. - - -UserRunMethodStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Executes a method after interacting with next button. Step type is expected to be used in scenarios where an action by the operator is required before running a method. - -.. code-block:: yaml - - steptype: UserRunMethodStep - step_name: Running the specified method - description: Runs the specified method - action_type: method - module: example_tests.py - input_mapping: - message: - type: direct - value: 'Find the specified file' - options: - type: direct - value: - - cancel: 'cancel' - - run: 'Run' - image_path: - type: direct - value: test.jpg - method_name: - type: method - value: 'is_PSU_disconnected' - argument1: - type: global - global_name: test - argument2: - type: local - local_name: testing - argument3: - type: direct - value: 5 - output_mapping: - output: - type: passfail - trigger_response: "run" - -* ``steptype`` (str): Determines the type of action. -* ``action_type`` (str): ``method``, ``read_attribute``, or ``write_attribute``. -* ``module`` (str): Python module path. With ``test_package``, it is relative to that package and may include nested directories. -* ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. -* ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. Multiple buttons can be added and cycled through by setting ``indexed`` to ``True``. - -These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. -* ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. -* ``method_name`` (dict): specifies the method to run. -* ``argument1-3`` (dict): Any dict that is not message, option or image path in input mapping will be considered input to method. multiple inputs are possible with them being ordered from top to bottom as inputs to the method. -* ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. -* ``trigger_response`` (str): you can choose the key to consider what button to push for a futher action. In this example, the key is "run". Keep the key similar to the specified value key in options. - -Example function of input would be for the step above. - -.. code-block:: python - - def simpleMethod(argument1, argument2, argument3) - #the input sequence seen from the above step. It shows the order of the - return - - -UserWriteStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Executes step to write values to variables or setting up the settings required for a comport. - -.. code-block:: yaml - - steptype: UserWriteStep - step_name: Writing a command - description: Write the ID or the port - input_mapping: - message: - type: direct - value: 'Write the ID or serial port of device' - image_path: - type: direct - value: test.jpg - options: - type: direct - value: - - 'cancel': 'cancel' - - 'ID': 'Write' - output_mapping: - output: - type: passfail - - -* ``steptype`` (str): Determines the type of action. -* ``input_mapping`` (dict): Inputs the desired period of time to wait before moving on to next step. -* ``message`` (dict): can write a message that is expected to be relevant for user to do before -* ``options`` (dict): options to add buttons. For the main functionality of this function, two keys are defined: ``'ID'`` for setting up a comport through gui and ``'wrt'`` for writing a string to a variable. - -These buttons on options are relevant for describing the action the button should take. The name of buttons are determined through key-value pairs as seen in the ``options``. It contains the keys 'yes' and two with each of their values. -The key ``cancel`` or ``'cancel'`` are both hardcoded to cancel a step and stop the entire test and can therefore not be used. keys do not need to be strings to operate unless the keys are ``yes`` or ``no`` as these are compiled to ``True`` and ``False``. - -* ``image_path`` (dict, optional): shows the specified image on gui during this step. Path is not required to find the image, as long as it is one layer deep inside the working directory. -* ``output_mapping`` (dict): outputs of the method. Each output(``arg``) is required to have a ``type``. - - -This step requires some local variables depending on which **key** is specified under ``options``. -If the **key** chosen is ``'ID'``, it requires the following local variables. - -* serial_ID -* serialport -* baudrate - -When the key is applied and button is pushed, a GUI pops up letting you choose baudrate and what comport that is available you want to connect to. you can send an IDN? command through button and when found to work it will save the values to the local variables. -The output mapping should just be pass/faill for this key. - -If the **key** chosen is ``'wrt'``, it requires an output mapping of either global or local scale to a variable. -The input written into the GUI window is sent to the output mapping to be saved as a ``str``. The following is the required ``output_mapping`` if the **key** is ``'wrt'`` on a button. - -.. code-block:: yaml - - output_mapping: - output: - type: local - local_name: example - - - -SerialNumberStep -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Prompts the operator for a device serial number via the GUI and stores it for the rest of the -recipe run. The serial number is written to both ``runtime.serial_number`` (used automatically -in all reports) and to the global variable ``serial_number`` so that subsequent steps can -reference it. - -Add this step wherever serial-number capture fits your workflow — typically as the first step -in the main sequence, but it can equally be placed inside a setup or sub-sequence. - -.. code-block:: yaml - - steptype: SerialNumberStep - step_name: Scan Serial Number - description: Ask the operator to scan or type the device serial number - input_mapping: {} - output_mapping: - serial_number: {type: global, global_name: serial_number} - -* ``steptype`` (str): Must be ``SerialNumberStep``. -* ``input_mapping`` (dict): No inputs required — leave as ``{}``. -* ``output_mapping`` (dict, optional): The step always stores the serial number in - ``runtime.serial_number`` and the ``serial_number`` global automatically. - Use the output mapping only when you additionally want to store the value - in a local variable or a differently-named global. - - .. code-block:: yaml - - # Store additionally in a local variable: - output_mapping: - serial_number: {type: local, local_name: device_sn} - -.. note:: - The GUI must handle the ``get_serial_number`` event and respond with the serial - number string on the provided response queue. See :doc:`gui_event_handling` for - details on wiring up the signal in your UI. - -**Minimal recipe example with serial number capture:** - -.. code-block:: yaml - - --- - name: Example Recipe - version: 1.0.0 - description: Recipe with serial number capture - main_sequence: Main - globals: - serial_number: null - - --- - sequence_name: Main - setup_steps: - - steptype: SerialNumberStep - step_name: Scan Serial Number - input_mapping: {} - output_mapping: {} - steps: - - steptype: PythonModuleStep - step_name: Run Tests - module: my_tests.py - action_type: method - method_name: run_all - input_mapping: {} - output_mapping: {} - teardown_steps: [] - locals: {} - parameters: {} - outputs: {} - - -Required globals and locals for certain steps. -================================================= - -To ensure the functionality of some of the step types, certain global and locals are required for different datatypes. This section explains which step requires what specific variables in the recipe. -The following steps that require global or local variables are found below. - -* **SSHConnectStep** - - Requires the ``cancel_key``, ``ssh_client``, ``host``, ``user``, ``port``, - and either ``password`` or ``private_key`` globals. - -* **UserLoadingStep** - - Requires the ``cancel_key`` and ``loadFile_key`` globals. - -* **UserRunMethodStep** - - Requires the ``cancel_key`` global. - -* **UserWriteStep** - - Requires the ``cancel_key``, ``ID_key``, and ``wrt_key`` globals. If - ``ID_key`` is specified in the options, it also requires the ``serial_ID``, - ``serialport``, and ``baudrate`` local variables. - -- **SerialNumberStep** - -Requires no global or local variables to be pre-declared. The step automatically -creates or updates the ``serial_number`` global with the value entered by the operator. -If you want to reference the serial number in subsequent steps, declare it in globals: - -.. code-block:: yaml - - globals: - serial_number: null - - - -.. _continue_on_error_details: - -Continue On Error Mechanism -============================ - -The framework can continue after ``ERROR`` results in non-critical steps. This -behavior applies regardless of whether ``recipe_version`` is present. - -Global Setting -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Set ``continue_on_error`` on an individual step, or in the recipe header to -override the value for every step: - -.. code-block:: yaml - - --- - name: My Recipe - continue_on_error: true - # ... other fields - -When ``continue_on_error`` is ``true``: -- Errors in steps marked as ``critical: false`` (default) will not stop sequence execution -- Errors in steps marked as ``critical: true`` will still stop sequence execution -- All errors are still logged and reported - -When the effective ``continue_on_error`` value is ``false`` (default): -- Any error in any step stops sequence execution (legacy behavior) -- The ``critical`` field has no effect - -Step-Level Critical Flag -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Individual steps can be marked as critical using the ``critical`` field: - -.. code-block:: yaml - - steps: - - steptype: PythonModuleStep - step_name: Optional Diagnostic Test - critical: false # Default - errors won't stop execution if continue_on_error is true - # ... other fields - - - steptype: PythonModuleStep - step_name: Essential Safety Check - critical: true # Errors will always stop execution - # ... other fields - -.. note:: - Notice the indentation inside ``steps`` and the ``-`` in front of the step. Adding this - is crucial for the functionality of the recipe. - - -Behavior Matrix -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The interaction between ``continue_on_error`` and ``critical`` settings: - -+-------------------+------------------+------------------------+ -| continue_on_error | step critical | Error Behavior | -+===================+==================+========================+ -| false | false (default) | Stop execution | -+-------------------+------------------+------------------------+ -| false | true | Stop execution | -+-------------------+------------------+------------------------+ -| true | false (default) | Continue execution | -+-------------------+------------------+------------------------+ -| true | true | Stop execution | -+-------------------+------------------+------------------------+ - -Use Cases -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This mechanism is useful for: - -- **Diagnostic Tests**: Run optional diagnostic steps that shouldn't fail the entire test if they encounter issues -- **Data Collection**: Continue gathering test data even if some measurements fail -- **Graceful Degradation**: Allow test sequences to complete as much as possible before stopping -- **Critical Safety Checks**: Ensure essential safety or validation steps always stop execution on failure - -Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: yaml - - --- - name: Hardware Test Suite - continue_on_error: true - - --- - sequence_name: Main - steps: - - steptype: PythonModuleStep - step_name: Initialize Hardware - critical: true # Setup failure should stop everything - # ... configuration - - - steptype: PythonModuleStep - step_name: Optional Calibration - critical: false # Calibration failure shouldn't stop the test - # ... configuration - - - steptype: PythonModuleStep - step_name: Core Functionality Test - critical: true # Main test failure should stop execution - # ... configuration - - - steptype: PythonModuleStep - step_name: Performance Metrics - critical: false # Metrics failure shouldn't stop cleanup - # ... configuration - - teardown_steps: - - steptype: PythonModuleStep - step_name: Hardware Cleanup - critical: true # Cleanup failure is critical for safety - # ... configuration - -In this example: -- If "Initialize Hardware" fails, execution stops immediately -- If "Optional Calibration" fails, execution continues to "Core Functionality Test" -- If "Core Functionality Test" fails, execution stops before "Performance Metrics" -- If "Performance Metrics" fails, execution continues to teardown -- If "Hardware Cleanup" fails, it's reported as a critical failure - -Recipe-level policy takes precedence over step-level settings: - -.. code-block:: yaml - - # Recipe-wide policy - continue_on_error: true - globals: {} - - # Step-level policy, used only when the recipe field is absent - # continue_on_error: true + literal: {type: direct, value: 12} + repeated: {type: direct, value: [1, 2], indexed: true} + from_local: {type: local, local_name: expected} + from_global: {type: global, global_name: target} + callback: {type: method, value: normalize} + +The exact input structures are :ref:`recipe-v2-input-direct`, +:ref:`recipe-v2-input-local`, :ref:`recipe-v2-input-global`, and +:ref:`recipe-v2-input-method`. Indexed direct inputs must contain lists, and +all indexed inputs on one step must have equal lengths. + +Output mappings also require an explicit ``type``. ``passfail``, ``equals``, +and ``range`` contribute verdicts; all configured verdict checks must pass. +``passthrough`` consumes an already-computed result and must be the only +verdict mapping on a step. ``local`` and ``global`` store values, while +``image`` publishes report images. See the generated +:ref:`recipe-v2-output-passfail` through :ref:`recipe-v2-output-range` +definitions for exact fields. + +Runtime-specific behavior +------------------------- + +``PythonModuleStep`` resolves ``module`` relative to ``test_package`` when the +header supplies that package. Without ``test_package``, the runtime uses its +file-based module lookup. A method action requires ``method_name``. + +``SequenceStep`` references another sequence in the same YAML stream. +``WaitStep`` requires a ``wait_time`` input. SSH steps require the connection +globals documented by the parser and must follow connect, upload, close order. +Exact step-specific fields are linked from the generated reference, beginning +with :ref:`recipe-v2-step-pythonmodulestep`. + +Error handling applies to execution errors, not failed verdicts. A header-level +``continue_on_error`` overrides step-level values when present; ``critical`` +errors still stop execution. Teardown steps run during sequence cleanup. + +Serialization and readable examples +------------------------------------ + +``recipe_to_yaml()`` accepts a validated aggregate recipe definition and +returns deterministic multi-document YAML text without performing file I/O. +It emits aliases and model defaults, omits ``None``, and always writes explicit +document separators. Parsing that output recreates an equal aggregate +definition. + +Serialization is formatting-destructive: comments, quoting choices, key +layout, and other source formatting are not retained. Maintained examples are +therefore kept readable by hand and may include comments. YamVIEW and +programmatic serialization produce normalized YAML instead. + +Migrating version 1 recipes +--------------------------- + +Version 1 is rejected; there is no automatic migration command or runtime +compatibility path. To migrate a file: + +1. Set the required header field to ``recipe_version: 2.0.0``. +2. Use exact canonical step names such as ``PythonModuleStep``, ``WaitStep``, + and ``UserInteractionStep``. Lowercase spellings are invalid. +3. Add ``type: direct`` to every literal input mapping that previously relied + on the implicit default. All input and output mappings are discriminated + explicitly. +4. Remove sequence-level ``serial_number``. Use ``SerialNumberStep`` and a + mapped global or local value when the run needs a serial number. +5. Add every now-required field, including header, sequence, and step + descriptions, and remove unknown fields. +6. Validate the complete multi-document file. Strict validation reports all + detected migration issues with paths and source positions; PyPTS never + changes legacy spelling or meaning silently. + +After migration, parse the source, serialize with ``recipe_to_yaml()``, and +parse again. Equal aggregate definitions establish semantic stability; byte +equality and preservation of source comments are intentionally not required. diff --git a/spikes/recipe_pydantic/__init__.py b/spikes/recipe_pydantic/__init__.py deleted file mode 100644 index 026d8a0..0000000 --- a/spikes/recipe_pydantic/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Historical spike namespace; production lives in :mod:`pypts`.""" - -from pypts.recipe_artifacts import render_json_schema -from pypts.recipe_language import Recipe -from pypts.recipe_parser import ( - Diagnostic, - ParseResult, - RecipeParseError, - SourcePosition, - SourceSpan, - dump_recipe, - parse_recipe_file, - parse_recipe_text, -) -from pypts.recipe_reference import render_reference - -__all__ = [ - "Diagnostic", - "ParseResult", - "Recipe", - "RecipeParseError", - "SourcePosition", - "SourceSpan", - "dump_recipe", - "parse_recipe_file", - "parse_recipe_text", - "render_json_schema", - "render_reference", -] diff --git a/src/pypts/YamVIEW/customGUIModules.py b/src/pypts/YamVIEW/customGUIModules.py index c1c0ae0..488f680 100644 --- a/src/pypts/YamVIEW/customGUIModules.py +++ b/src/pypts/YamVIEW/customGUIModules.py @@ -397,12 +397,12 @@ def generate_template_yaml(self, data): # Generate YAML from pypts.recipe_language import Recipe as RecipeDefinition - from pypts.recipe_parser import dump_recipe + from pypts.recipe_parser import recipe_to_yaml definition = RecipeDefinition.model_validate( {"header": header, "sequences": [sequence]} ) - yaml_body = dump_recipe(definition) + yaml_body = recipe_to_yaml(definition) # Combine SPDX header and YAML body yaml_string = spdx_header + yaml_body diff --git a/src/pypts/YamVIEW/recipe_creator.py b/src/pypts/YamVIEW/recipe_creator.py index 4ba8ab1..a81e8c7 100644 --- a/src/pypts/YamVIEW/recipe_creator.py +++ b/src/pypts/YamVIEW/recipe_creator.py @@ -38,7 +38,7 @@ import webbrowser from pypts.YamVIEW.styles import * from pypts.YamVIEW.verify_recipe import * -from pypts.recipe_parser import dump_recipe, parse_recipe_text +from pypts.recipe_parser import parse_recipe_text, recipe_to_yaml import sys from PySide6.QtGui import QColor, QTextCharFormat, QFont from PySide6.QtCore import Qt @@ -551,7 +551,7 @@ def on_save_as_clicked(self): # Extract data from the text view try: # data = self.extract_treeView_to_data() - data = dump_recipe( + data = recipe_to_yaml( parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() ) except Exception as e: @@ -600,7 +600,7 @@ def on_save_clicked(self): # Extract data from text view try: # data = self.extract_treeView_to_data() - data = dump_recipe( + data = recipe_to_yaml( parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() ) except Exception as e: diff --git a/src/pypts/example_commented_recipes/readme.md b/src/pypts/example_commented_recipes/readme.md deleted file mode 100644 index 62959ee..0000000 --- a/src/pypts/example_commented_recipes/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later -Here we store the recipes that are used to verify the functionality of the framework. -All of the recipes should finish execution without a crash \ No newline at end of file diff --git a/src/pypts/example_commented_recipes/simple_multiplestep_recipe.yml b/src/pypts/example_commented_recipes/simple_multiplestep_recipe.yml deleted file mode 100644 index 5eb5028..0000000 --- a/src/pypts/example_commented_recipes/simple_multiplestep_recipe.yml +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later - -# This section is related to the whole recipe. -# We could define the name, version and description herem as well as the -# continue_on_error flag, which could abort tests on failure -name: Example Test Recipe -version: 0.1.0 -recipe_version: 1.0.0 -description: A sample recipe demonstrating how to call functions from example_tests.py -main_sequence: Main -continue_on_error: true -globals: ---- -# Recipe can contain multiple sequences. Essentially, sequence is just a group of steps -sequence_name: Main -description: The main sequence of steps for the example recipe. -# we can define sequence - scope variables, that can be used by other steps as inputs -parameters: - target_value: '0' -locals: - target_value: '45' - test_name: Hello - wonderful: None - wunderbar: None -outputs: - my_output: None -setup_steps: [] -# we define multiple step types. Following implementations can be treated as examples -steps: -- steptype: UserInteractionStep - step_name: Are you all right? - description: Asking user for confirmation - skip: false - input_mapping: - message: - type: direct - value: 'Hello there. Are you doing all right? - Please let us know by pressing the correct button :) - ' - image_path: - type: direct - value: lego2.jpg - options: - type: direct - value: - - 'yes': '' - - 'no': '' - output_mapping: - output: - type: equals - value: 'yes' -- steptype: PythonModuleStep - step_name: other_test - description: Python existing method - expect pass - action_type: method - module: example_tests - method_name: other_test - input_mapping: {} - output_mapping: - some_return: - type: passfail - value: - type: local - local_name: some_value -- steptype: PythonModuleStep - step_name: other_testa - description: Python non-existing method - expect error - action_type: method - module: example_testsa - method_name: other_testa - input_mapping: {} - output_mapping: - some_return: - type: passfail - value: - type: local - local_name: test_value -- steptype: PythonModuleStep - step_name: simple_output - description: Python method, while providing non-existing inputs - expect error - action_type: method - module: example_tests.py - method_name: simple_output - input_mapping: - value: - type: local - local_name: test_value - output_mapping: - my_output: - type: local - local_name: calculated_output -- steptype: PythonModuleStep - step_name: test_to_run (Check value 45) - description: Python existing method - invalid range - action_type: method - module: example_tests.py - method_name: test_to_run - input_mapping: - target: - type: direct - value: '45' - output_mapping: - compare: - type: passfail - other_output: - type: local - local_name: some_string -- steptype: PythonModuleStep - step_name: range_test (valid Range) - description: Python existing method - invalid range - action_type: method - module: example_tests.py - method_name: range_test - input_mapping: - value: - type: direct - value: '15' - min: - type: direct - value: '10' - max: - type: direct - value: '20' - output_mapping: - compare: - type: passfail -- steptype: WaitStep - step_name: Wait for 3s - description: Waiting 3 seconds to let user check confirm - skip: true - input_mapping: - wait_time: - value: '3' - output_mapping: {} -- steptype: PythonModuleStep - step_name: Run range_test (outside Range) - description: Some description - action_type: method - module: example_tests.py - method_name: range_test - input_mapping: - value: - type: direct - value: '25' - min: - type: direct - value: '10' - max: - type: direct - value: '20' - output_mapping: - compare: - type: passfail -- steptype: PythonModuleStep - step_name: Generate Error Deliberately - description: Some description - action_type: method - module: example_tests.py - method_name: generate_error - input_mapping: {} - output_mapping: {} -teardown_steps: [] diff --git a/src/pypts/examples/environment_setup_tools/Minimal_setup/Minimal_setup_recipe.yml b/src/pypts/examples/environment_setup_tools/Minimal_setup/Minimal_setup_recipe.yml index acd402f..cdf443c 100644 --- a/src/pypts/examples/environment_setup_tools/Minimal_setup/Minimal_setup_recipe.yml +++ b/src/pypts/examples/environment_setup_tools/Minimal_setup/Minimal_setup_recipe.yml @@ -3,15 +3,15 @@ # SPDX-License-Identifier: LGPL-2.1-or-later # This section is related to the whole recipe. -# We could define the name, version and description herem as well as the -# continue_on_error flag, which could abort tests on failure +# It defines the name, versions, description, and execution policy. name: Minimal setup Recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe demonstrating how to call functions from example_tests.py in a minimal pypts setup main_sequence: Main -globals: - #continue_on_error: False #Overall continue_on_error. If it exists, then it will overwrite specific step specifications for continue_on_error +# Optional; when omitted, each step uses its own continue_on_error setting. +# continue_on_error: false +globals: global_value: 45 loadFile_key: 'file' cancel_key: 'cancel' #key used for buttons that wishes to stop steps. @@ -156,6 +156,7 @@ steps: skip: true input_mapping: wait_time: + type: direct value: '3' output_mapping: {} - steptype: PythonModuleStep @@ -208,3 +209,4 @@ steps: teardown_steps: - steptype: SSHCloseStep step_name: Closes the SSH client + description: Close the SSH connection opened during setup diff --git a/src/pypts/examples/environment_setup_tools/Package_based_setup/Package_based_recipe.yml b/src/pypts/examples/environment_setup_tools/Package_based_setup/Package_based_recipe.yml index b64f8ef..923d82e 100644 --- a/src/pypts/examples/environment_setup_tools/Package_based_setup/Package_based_recipe.yml +++ b/src/pypts/examples/environment_setup_tools/Package_based_setup/Package_based_recipe.yml @@ -3,17 +3,17 @@ # SPDX-License-Identifier: LGPL-2.1-or-later # this section is related to the whole recipe. -# we could define the name, version and description herem as well as the -# continue_on_error flag, which could abort tests on failure +# it defines the name, versions, description, and execution policy. name: package_based recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: a sample recipe demonstrating how to call functions from example_tests.py in a minimal pypts setup main_sequence: main test_package: example_package +# optional; when omitted, each step uses its own continue_on_error setting. +# continue_on_error: false globals: - #continue_on_error: false #overall continue_on_error. if it exists, then it will overwrite specific step specifications for continue_on_error global_value: 45 loadfile_key: 'file' cancel_key: 'cancel' #key used for buttons that wishes to stop steps. @@ -42,14 +42,14 @@ outputs: setup_steps: #creates a threading holding the opened ssh connection so it can be called and used elsewhere. - - steptype: sshconnectstep + - steptype: SSHConnectStep step_name: find the ssh client connection description: asking user to find the file needed continue_on_error: false skip: true # we define multiple step types. following implementations can be treated as examples steps: - - steptype: userinteractionstep + - steptype: UserInteractionStep step_name: are you all right? description: asking user for confirmation skip: false @@ -70,7 +70,7 @@ steps: output: type: equals value: 'yes' - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: other_test description: python existing method - expect pass action_type: method @@ -83,7 +83,7 @@ steps: value: type: local local_name: some_value - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: other_testa description: python non-existing method - expect error action_type: method @@ -98,7 +98,7 @@ steps: local_name: test_value continue_on_error: true - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: simple_output description: python method, while providing non-existing inputs - expect error @@ -115,7 +115,7 @@ steps: local_name: calculated_output continue_on_error: true - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: test_to_run (check value 45) description: python existing method - invalid range action_type: method @@ -133,7 +133,7 @@ steps: local_name: some_string continue_on_error: true - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: range_test (valid range) description: python existing method - invalid range action_type: method @@ -152,15 +152,16 @@ steps: output_mapping: compare: type: passfail - - steptype: waitstep + - steptype: WaitStep step_name: wait for 3s description: waiting 3 seconds to let user check confirm skip: true input_mapping: wait_time: + type: direct value: '3' output_mapping: {} - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: run range_test (outside range) description: some description action_type: method @@ -179,7 +180,7 @@ steps: output_mapping: compare: type: passfail - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: generate error deliberately description: some description action_type: method @@ -187,10 +188,10 @@ steps: method_name: generate_error input_mapping: {} output_mapping: {} - - steptype: sshclosestep + - steptype: SSHCloseStep step_name: sshclose description: describe the test in this box - - steptype: pythonmodulestep + - steptype: PythonModuleStep step_name: sinewave test (60 hz) description: generate a 60 hz sinewave, validate it via fft, and attach the plot to the report action_type: method @@ -209,5 +210,6 @@ steps: chart: type: image teardown_steps: - - steptype: sshclosestep - step_name: closes the ssh client \ No newline at end of file + - steptype: SSHCloseStep + step_name: closes the ssh client + description: close the ssh connection opened during setup diff --git a/src/pypts/recipe_parser.py b/src/pypts/recipe_parser.py index 680b6b5..fafaa73 100644 --- a/src/pypts/recipe_parser.py +++ b/src/pypts/recipe_parser.py @@ -525,10 +525,17 @@ def parse_recipe_file(path: str | Path, encoding: str = "utf-8") -> ParseResult: return parse_recipe_text(text, str(source_path)) -def dump_recipe(recipe: Recipe) -> str: - """Serialize a typed recipe as canonical multi-document YAML.""" +def recipe_to_yaml(recipe: Recipe) -> str: + """Return deterministic multi-document YAML for a validated aggregate recipe. + + This function performs no file I/O. It emits field aliases and model + defaults, excludes ``None`` values, and uses explicit YAML document + separators. Comments, quoting choices, and other source formatting are not + preserved. Parsing the returned text produces an aggregate definition + semantically equal to the validated input. + """ if not isinstance(recipe, Recipe): - raise TypeError("dump_recipe expects a Recipe") + raise TypeError("recipe_to_yaml expects a Recipe") documents = [ recipe.header.model_dump(mode="python", by_alias=True, exclude_none=True), *[ diff --git a/src/pypts/recipes/Instrument_test.yml b/src/pypts/recipes/Instrument_test.yml index e3329a3..507c8ed 100644 --- a/src/pypts/recipes/Instrument_test.yml +++ b/src/pypts/recipes/Instrument_test.yml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later name: Instrument Example Use Test recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe used to run instruments through the inputs given. main_sequence: Instrument_run # continue_on_error: False # Overall policy. When present, it overrides step-level settings. @@ -40,6 +40,7 @@ steps: skip: false input_mapping: wait_time: + type: direct value: '3' #Measure from the CNT91 device and read back the values. The input is for the visa device, aka the ID and device type. @@ -66,6 +67,7 @@ steps: skip: false input_mapping: wait_time: + type: direct value: '3' #This test pings the device through using the basic instrument class to showcase that even the normal instrument class operates as planned diff --git a/src/pypts/recipes/RTM_recipe.yml b/src/pypts/recipes/RTM_recipe.yml index 606518c..0ec76ed 100644 --- a/src/pypts/recipes/RTM_recipe.yml +++ b/src/pypts/recipes/RTM_recipe.yml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later name: Example Test Recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe demonstrating how to call functions from example_tests.py main_sequence: Main continue_on_error: true diff --git a/src/pypts/recipes/RTM_recipe_shared_object.yml b/src/pypts/recipes/RTM_recipe_shared_object.yml index 68a2b7f..d423746 100644 --- a/src/pypts/recipes/RTM_recipe_shared_object.yml +++ b/src/pypts/recipes/RTM_recipe_shared_object.yml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: LGPL-2.1-or-later name: Example Test Recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe demonstrating how to call functions from example_tests.py main_sequence: Main continue_on_error: true diff --git a/src/pypts/recipes/black_forest.yml b/src/pypts/recipes/black_forest.yml index 459bbe6..b527607 100644 --- a/src/pypts/recipes/black_forest.yml +++ b/src/pypts/recipes/black_forest.yml @@ -4,7 +4,7 @@ name: black forest cake testing version: 0.0.1 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: preparing the cake and checking the flavours main_sequence: my beautiful sequence continue_on_error: true @@ -17,7 +17,7 @@ locals: {} outputs: {} setup_steps: [] steps: -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 1 description: step 1 will be skipped skip: true @@ -34,7 +34,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 2 description: step 2 will be skipped skip: true @@ -51,7 +51,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 3 description: step 3 will show a message with yes/no decision to the user skip: false @@ -68,7 +68,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 4 description: step 4 will show a message and image to the user skip: false @@ -85,7 +85,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 5 description: step 5 will execute a pythonmodule script skip: false @@ -102,7 +102,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 6 description: step 6 will fail, because of unhandled excep[tion in the user code skip: false @@ -119,7 +119,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 7 description: step 7 will take the output from test5 and pass/fail accordingly skip: false @@ -136,7 +136,7 @@ steps: output: type: equals value: 'yes' -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 8 description: step 8 will take output from (crashed) test 6 and pass/fail accordingly skip: false diff --git a/src/pypts/recipes/comprehensive_recipe.yml b/src/pypts/recipes/comprehensive_recipe.yml index b63338b..45dbbae 100644 --- a/src/pypts/recipes/comprehensive_recipe.yml +++ b/src/pypts/recipes/comprehensive_recipe.yml @@ -3,11 +3,11 @@ # SPDX-License-Identifier: LGPL-2.1-or-later name: Comprehensive Example Test recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe demonstrating many different combinations of input/outputs. main_sequence: Comprehensive_IO_Types test_package: pypts -report_name_include_serial: True #Setting this to true will add the serial number to the report name. This is useful when running multiple tests on the same day to easily differentiate them. +report_name_include_serial: True # Optional; defaults to false. True adds the serial number to the report name. # continue_on_error: False # Overall policy. When present, it overrides step-level settings. globals: global_value: 45 @@ -70,6 +70,7 @@ steps: skip: false input_mapping: wait_time: + type: direct value: '3' @@ -500,6 +501,7 @@ steps: skip: false input_mapping: wait_time: + type: direct value: '3' output_mapping: {} diff --git a/src/pypts/recipes/graph_testing.yml b/src/pypts/recipes/graph_testing.yml index 88673b7..0d3c29d 100644 --- a/src/pypts/recipes/graph_testing.yml +++ b/src/pypts/recipes/graph_testing.yml @@ -4,7 +4,7 @@ name: recipe designed for the graph testing version: 0.0.1 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: satart acquisition and load the graph into the gui main_sequence: main continue_on_error: true @@ -17,7 +17,7 @@ locals: {} outputs: {} setup_steps: [] steps: -- steptype: userinteractionstep +- steptype: UserInteractionStep step_name: step 1 description: step 1 will be skipped skip: true @@ -34,7 +34,7 @@ steps: output: type: equals value: 'yes' -- steptype: pythonmodulestep +- steptype: PythonModuleStep step_name: load_existing_graph description: loading a file that already contains data action_type: method @@ -48,7 +48,7 @@ steps: type: direct value: "/home/pts/dev/pypts/src/pypts/XYGraph/123.csv" output_mapping: {} -- steptype: pythonmodulestep +- steptype: PythonModuleStep step_name: simulate_sine_wave description: Simulates acquisition action_type: method diff --git a/src/pypts/recipes/simple_recipe.yml b/src/pypts/recipes/simple_recipe.yml index 6ebc1f9..057152c 100644 --- a/src/pypts/recipes/simple_recipe.yml +++ b/src/pypts/recipes/simple_recipe.yml @@ -3,16 +3,15 @@ # SPDX-License-Identifier: LGPL-2.1-or-later name: Example Test Recipe version: 0.1.0 -recipe_version: 1.0.0 +recipe_version: 2.0.0 description: A sample recipe demonstrating how to call functions from example_tests.py main_sequence: Main -report_name_include_serial: True +report_name_include_serial: True # Optional; defaults to false. continue_on_error: true globals: {} --- sequence_name: Main description: The main sequence of steps for the example recipe. -serial_number: 12 parameters: target_value: '0' locals: @@ -161,6 +160,7 @@ steps: skip: true input_mapping: wait_time: + type: direct value: '3' output_mapping: {} - steptype: PythonModuleStep diff --git a/src/pypts/recipes/subsequence_executions_draft.yml b/src/pypts/recipes/subsequence_executions_draft.yml deleted file mode 100644 index c6932b6..0000000 --- a/src/pypts/recipes/subsequence_executions_draft.yml +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later - -# todo - test it, build documentation on the subsequences -# todo - the subsequence execution was never actually tested yet - -# if this sequence is run as a subsequence -# -# -# -# - steptype: SequenceStep -# step_name: Testing a subsequence -# sequence: {type: internal, name: subsequence_test} -# input_mapping: -# sublocal: {type: direct, value: 45} -# output_mapping: -# hello: {type: local, local_name: hello} -# seq_result: {type: passthrough} -#teardown_steps: [] -# -#--- -#sequence_name: Voltage subtests -#description: Set of tests to verify the voltages of 12V, 3.3V and 5V rails -#parameters: -# target_value: 0 -#locals: -# 12V_tolerance: 5 -# 5V_tolerance: 5 -# 3P3V_tolerance: 5 -#outputs: [] -# -#setup_steps: [] -#steps: -# - steptype: UserInteractionStep -# step_name: subsequence_test -# skip: true -# input_mapping: -# message: {type: direct, value: Hello there} -# image_path: {type: direct, value: images\lego2.jpg} -# options: {type: direct, value: ["yes", "no"]} -# output_mapping: -# output: {type: equals, value: "yes"} -# -#teardown_steps: [] -## - steptype: SequenceStep -## step_name: Testing a subsequence -## sequence: {type: internal, name: Subsequence} -## input_mapping: -## sublocal: {type: direct, value: 45} -## output_mapping: -## hello: {type: local, local_name: hello} -## seq_result: {type: passthrough} -## -## -##--- -##sequence_name: Subsequence -##description: -## -##parameters: # list of local variable names. These are the ones that the calling step can set before running the subsequence -## - sublocal -## -##locals: -## sublocal: 34 -## hello: bla -## bye: nope -## -##outputs: [sublocal, hello] -## -##setup_steps: [] -##steps: -## -##- steptype: UserInteractionStep -## step_name: Ask user if he/she is doing all right -## skip: true -## input_mapping: -## message: {type: direct, value: Hello there} -## image_path: {type: direct, value: images\lego2.jpg} -## options: {type: direct, value: ["yes", "no"]} -## output_mapping: -## output: {type: equals, value: "yes"} -## -##- steptype: PythonModuleStep -## step_name: Run test_to_run function -## action_type: method -## module: C:\dev\pypts\test_dut.py -## method_name: test_to_run -## input_mapping: -## target: {type: local, local_name: sublocal} -## output_mapping: -## compare: {type: passfail} -## other_output: {type: local, local_name: hello} -## -##teardown_steps: [] \ No newline at end of file diff --git a/tests/functional_tests/test_recipes_format.py b/tests/functional_tests/test_recipes_format.py index 1c83116..a0bddd6 100644 --- a/tests/functional_tests/test_recipes_format.py +++ b/tests/functional_tests/test_recipes_format.py @@ -8,4 +8,4 @@ def test_recipes_format(): recipe_path = get_project_root() / "src" / "pypts" / "recipes" - assert not validate_all_recipes_in_folder(recipe_path) + assert validate_all_recipes_in_folder(recipe_path) diff --git a/tests/unit_tests/test_environment_setup_recipes.py b/tests/unit_tests/test_environment_setup_recipes.py new file mode 100644 index 0000000..a25c5e3 --- /dev/null +++ b/tests/unit_tests/test_environment_setup_recipes.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later +"""Environment setup commands copy valid recipe-language 2 templates.""" + +from pypts.examples.environment_setup_tools.Minimal_setup import init_env_min +from pypts.examples.environment_setup_tools.Package_based_setup import init_env_pack +from pypts.recipe_parser import parse_recipe_file + + +def test_minimal_setup_copies_a_valid_v2_recipe(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + init_env_min.main() + + copied = tmp_path / "Minimal_setup_recipe.yml" + result = parse_recipe_file(copied) + assert result.is_valid, result.errors + assert result.require_recipe().header.recipe_version == "2.0.0" + + +def test_package_setup_copies_a_valid_v2_recipe(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(init_env_pack.subprocess, "check_call", lambda command: None) + init_env_pack.main() + + copied = ( + tmp_path / "src" / "example_package" / "resources" / "Package_based_recipe.yml" + ) + result = parse_recipe_file(copied) + assert result.is_valid, result.errors + assert result.require_recipe().header.recipe_version == "2.0.0" diff --git a/tests/unit_tests/test_recipe.py b/tests/unit_tests/test_recipe.py index caab55c..b14981e 100644 --- a/tests/unit_tests/test_recipe.py +++ b/tests/unit_tests/test_recipe.py @@ -3,6 +3,7 @@ """Typed recipe-to-runtime construction and execution tests.""" import queue +from pathlib import Path import pytest from pydantic import TypeAdapter @@ -18,7 +19,10 @@ ) from pypts.recipe_language import STEP_DEFINITION_MODELS, StepDefinition from pypts.recipe_language import Recipe as RecipeDefinition -from pypts.recipe_parser import RecipeParseError, dump_recipe +from pypts.recipe_parser import RecipeParseError, recipe_to_yaml + +ROOT = Path(__file__).parents[2] +BUNDLED_RECIPES = sorted((ROOT / "src" / "pypts" / "recipes").glob("*.yml")) def step_example(kind, **updates): @@ -141,7 +145,7 @@ def test_recipe_from_definition_constructs_typed_runtime_state_without_reparse() def test_recipe_path_requires_v2_and_raises_structured_error(tmp_path): v2 = tmp_path / "v2.yml" - v2.write_text(dump_recipe(definition()), encoding="utf-8") + v2.write_text(recipe_to_yaml(definition()), encoding="utf-8") assert Recipe(v2).main_sequence == "Main" legacy = tmp_path / "v1.yml" @@ -151,6 +155,30 @@ def test_recipe_path_requires_v2_and_raises_structured_error(tmp_path): assert "unsupported-recipe-version" in {d.code for d in caught.value.diagnostics} +@pytest.mark.parametrize("path", BUNDLED_RECIPES) +def test_every_bundled_recipe_constructs_runtime_state(path): + recipe = Recipe(path) + assert recipe.definition.header.recipe_version == "2.0.0" + assert recipe.main_sequence in recipe.sequences + + +@pytest.mark.parametrize("path", BUNDLED_RECIPES) +def test_every_bundled_recipe_enters_execution_without_hardware(path, monkeypatch): + monkeypatch.setattr(pypts.recipe.time, "sleep", lambda seconds: None) + monkeypatch.setattr( + pypts.recipe.ExecutableSequenceStep, + "run", + lambda self, active_runtime, inputs, stop_event: ResultType.DONE, + ) + active_runtime = runtime() + recipe = Recipe(path) + + assert recipe.run(active_runtime) == [] + assert active_runtime.recipe_name == recipe.name + assert active_runtime.recipe_file_name == path.name + Runtime.stop_event.clear() + + def test_registry_exactly_matches_step_definition_discriminators(): discriminators = { model.model_fields["steptype"].examples[0] diff --git a/tests/unit_tests/test_recipe.yaml b/tests/unit_tests/test_recipe.yaml deleted file mode 100644 index 07b97f8..0000000 --- a/tests/unit_tests/test_recipe.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later - ---- -name: Example Test Recipe -version: 0.1.0 -description: A sample recipe demonstrating how to call functions from example_tests.py -main_sequence: Main -test_package: pypts -globals: {} - ---- -sequence_name: Main -description: The main sequence of steps for the example recipe. -setup_steps: [] -steps: - - steptype: PythonModuleStep - step_name: Run Other Test - action_type: method - module: example_tests.py - method_name: other_test - input_mapping: {} - output_mapping: - some_return: {type: passfail} - value: {type: local, local_name: test_value} - - - steptype: PythonModuleStep - step_name: Run Simple Output Test - action_type: method - module: example_tests.py - method_name: simple_output - input_mapping: - value: {type: local, local_name: test_value} - output_mapping: - my_output: {type: local, local_name: calculated_output} # Store the output - - - steptype: PythonModuleStep - step_name: Run Test To Run (Check 45) - action_type: method - module: example_tests.py - method_name: test_to_run - input_mapping: - target: {type: direct, value: 45} - output_mapping: - compare: {type: passfail} - other_output: {type: local, local_name: some_string} # Store the other output - - - steptype: PythonModuleStep - step_name: Run Range Test (Inside Range) - action_type: method - module: example_tests.py - method_name: range_test - input_mapping: - value: {type: direct, value: 15} - min: {type: direct, value: 10} - max: {type: direct, value: 20} - output_mapping: - compare: {type: range, min: 10, max: 20} # Should pass - - - steptype: PythonModuleStep - step_name: Run Range Test (Outside Range) - action_type: method - module: example_tests.py - method_name: range_test - input_mapping: - value: {type: direct, value: 25} - min: {type: direct, value: 10} - max: {type: direct, value: 20} - output_mapping: - compare: {type: range, min: 10, max: 20} # Should fail - - # Uncomment the following step to test error handling - # - steptype: PythonModuleStep - # step_name: Generate Error Deliberately - # action_type: method - # module: src/pypts/example_tests.py - # method_name: generate_error - # input_mapping: {} - # output_mapping: {} # No specific output check needed, framework handles the error - -teardown_steps: [] -parameters: {} -locals: - test_value: null # Initialize local variable - calculated_output: null - some_string: "" -outputs: {} diff --git a/tests/unit_tests/test_recipe_language.py b/tests/unit_tests/test_recipe_language.py index 08a29d1..1cc5dfc 100644 --- a/tests/unit_tests/test_recipe_language.py +++ b/tests/unit_tests/test_recipe_language.py @@ -12,7 +12,7 @@ import yaml from pydantic import ValidationError -from pypts import recipe_language +from pypts import recipe_language, recipe_parser from pypts.recipe_artifacts import ( DEFAULT_REFERENCE_PATH, DEFAULT_SCHEMA_PATH, @@ -25,14 +25,17 @@ ) from pypts.recipe_parser import ( RecipeParseError, - dump_recipe, parse_recipe_file, parse_recipe_text, + recipe_to_yaml, ) from pypts.recipe_reference import render_reference ROOT = Path(__file__).parents[2] RECIPES = ROOT / "src" / "pypts" / "recipes" +SETUP_TEMPLATES = ROOT / "src" / "pypts" / "examples" / "environment_setup_tools" +MAINTAINED_RECIPES = sorted(RECIPES.glob("*.yml")) +MAINTAINED_TEMPLATES = sorted(SETUP_TEMPLATES.glob("*/*.yml")) def test_step_definition_names_are_the_only_public_union_and_model_registry(): @@ -149,7 +152,7 @@ def recipe_for_step(step): def test_every_step_validates_serializes_and_reparses(model): first = parse_recipe_text(recipe_for_step(STEP_EXAMPLES[model.__name__])) assert first.is_valid, first.errors - second = parse_recipe_text(dump_recipe(first.require_recipe())) + second = parse_recipe_text(recipe_to_yaml(first.require_recipe())) assert second.is_valid, second.errors assert second.recipe == first.recipe @@ -169,7 +172,7 @@ def test_every_input_mapping_validates_serializes_and_reparses(model): } first = parse_recipe_text(recipe_for_step(step)) assert isinstance(first.require_recipe().sequences[0].steps[0].input_mapping["example"], model) - assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + assert parse_recipe_text(recipe_to_yaml(first.require_recipe())).recipe == first.recipe OUTPUT_EXAMPLES = { @@ -190,7 +193,7 @@ def test_every_output_mapping_validates_serializes_and_reparses(model): } first = parse_recipe_text(recipe_for_step(step)) assert isinstance(first.require_recipe().sequences[0].steps[0].output_mapping["example"], model) - assert parse_recipe_text(dump_recipe(first.require_recipe())).recipe == first.recipe + assert parse_recipe_text(recipe_to_yaml(first.require_recipe())).recipe == first.recipe def test_defaults_are_typed_dumped_and_models_are_frozen(): @@ -198,12 +201,32 @@ def test_defaults_are_typed_dumped_and_models_are_frozen(): step = recipe.sequences[0].steps[0] assert step.skip is step.critical is step.continue_on_error is False assert recipe.header.report == "overwrite" - dumped = dump_recipe(recipe) + dumped = recipe_to_yaml(recipe) assert "report: overwrite" in dumped and "skip: false" in dumped with pytest.raises(ValidationError): step.skip = True +def test_recipe_to_yaml_is_deterministic_explicit_and_formatting_destructive(): + assert recipe_parser.recipe_to_yaml is recipe_to_yaml + assert not hasattr(recipe_parser, "dump_recipe") + text = recipe_for_step(STEP_EXAMPLES["UserInteractionStep"]) + commented = "# This comment is intentionally not retained.\n" + text + recipe = parse_recipe_text(commented).require_recipe() + + first = recipe_to_yaml(recipe) + second = recipe_to_yaml(recipe) + + assert first == second + assert first.startswith("---\n") + assert first.count("---\n") == 2 + assert "report: overwrite" in first + assert "# This comment" not in first + assert parse_recipe_text(first).require_recipe() == recipe + with pytest.raises(TypeError, match="recipe_to_yaml expects a Recipe"): + recipe_to_yaml({}) + + def test_strict_types_unknown_fields_and_structural_rules_are_rejected(): bad = STEP_EXAMPLES["PythonModuleStep"] | { "skip": 0, @@ -278,6 +301,14 @@ def test_v1_migration_errors_are_aggregated_across_documents(): assert all(item.source_name == "legacy.yml" and item.span is not None for item in result.errors) +def test_missing_recipe_version_has_a_clear_diagnostic(): + missing_version = header() + del missing_version["recipe_version"] + result = parse_recipe_text(source(missing_version, sequence())) + finding = next(item for item in result.errors if item.path[-1:] == ("recipe_version",)) + assert finding.code == "missing-field" + + def test_source_spans_point_to_fields_and_nearest_parent(): text = source(header(main_sequence="Missing"), sequence()) result = parse_recipe_text(text, "broken.yml") @@ -359,28 +390,21 @@ def test_ssh_context_and_ordering_are_semantic_rules(): } -@pytest.mark.parametrize( - "path", - sorted( - path for path in RECIPES.glob("*.yml") - if path.name != "subsequence_executions_draft.yml" - ), -) -def test_bundled_corpus_is_rejected_until_phase_7(path): +@pytest.mark.parametrize("path", MAINTAINED_RECIPES + MAINTAINED_TEMPLATES) +def test_every_maintained_recipe_and_setup_template_is_valid_v2(path): result = parse_recipe_file(path) - assert not result.is_valid - assert "unsupported-recipe-version" in {item.code for item in result.errors} - - -def test_raw_legacy_corpus_exposes_migration_diagnostics(): - results = [ - parse_recipe_file(path) - for path in RECIPES.glob("*.yml") - if path.name != "subsequence_executions_draft.yml" - ] - assert all("unsupported-recipe-version" in {item.code for item in result.errors} for result in results) - all_codes = {item.code for result in results for item in result.errors} - assert {"noncanonical-step-type", "missing-input-type", "removed-sequence-field"} <= all_codes + assert result.is_valid, result.errors + assert not result.diagnostics + assert result.require_recipe().header.recipe_version == "2.0.0" + + +@pytest.mark.parametrize("path", MAINTAINED_RECIPES) +def test_every_maintained_recipe_is_semantically_stable_when_serialized(path): + first = parse_recipe_file(path).require_recipe() + serialized = recipe_to_yaml(first) + second = parse_recipe_text(serialized, f"serialized:{path.name}") + assert second.is_valid, second.errors + assert second.require_recipe() == first def test_generated_schema_and_reference_are_complete_and_current(): diff --git a/tests/unit_tests/test_recipe_pydantic_docs.py b/tests/unit_tests/test_recipe_pydantic_docs.py index 57a3b52..3ae80d7 100644 --- a/tests/unit_tests/test_recipe_pydantic_docs.py +++ b/tests/unit_tests/test_recipe_pydantic_docs.py @@ -8,6 +8,8 @@ import json from pathlib import Path +import pytest + from pypts.recipe_artifacts import ( main, render_json_schema, @@ -15,14 +17,14 @@ write_artifacts, ) from pypts.recipe_parser import ( - dump_recipe, parse_recipe_file, parse_recipe_text, + recipe_to_yaml, ) from pypts.recipe_reference import render_reference ROOT = Path(__file__).parents[2] -DOC_RECIPE = ROOT / "docs" / "source" / "_examples" / "recipe_v2.yml" +DOC_RECIPES = sorted((ROOT / "docs" / "source" / "_examples").glob("*.yml")) ARCHITECTURE = ROOT / "docs" / "source" / "recipe_language_architecture.rst" MAINTENANCE = ROOT / "docs" / "source" / "recipe_language_maintenance.rst" REFERENCE_RENDERER = ROOT / "src" / "pypts" / "recipe_reference.py" @@ -96,7 +98,6 @@ def test_json_only_renderer_has_no_model_runtime_ui_or_sphinx_imports(): imported.add(node.module) forbidden = { "pydantic", - "spikes.recipe_pydantic.models", "pypts.recipe", "pypts.steps", "pypts.YamVIEW", @@ -109,11 +110,12 @@ def test_json_only_renderer_has_no_model_runtime_ui_or_sphinx_imports(): ) -def test_documentation_recipe_is_warning_free_and_model_stable(): - first = parse_recipe_file(DOC_RECIPE) +@pytest.mark.parametrize("path", DOC_RECIPES) +def test_documentation_recipes_are_warning_free_and_model_stable(path): + first = parse_recipe_file(path) assert first.is_valid, first.errors assert not first.warnings - second = parse_recipe_text(dump_recipe(first.require_recipe()), "canonical:recipe_v2.yml") + second = parse_recipe_text(recipe_to_yaml(first.require_recipe()), f"serialized:{path.name}") assert second.is_valid, second.errors assert not second.warnings assert second.recipe == first.recipe diff --git a/tests/unit_tests/test_verify_recipe.py b/tests/unit_tests/test_verify_recipe.py index 5b4a676..f656346 100644 --- a/tests/unit_tests/test_verify_recipe.py +++ b/tests/unit_tests/test_verify_recipe.py @@ -3,7 +3,7 @@ import pytest -from pypts.recipe_parser import dump_recipe, parse_recipe_file +from pypts.recipe_parser import parse_recipe_file, recipe_to_yaml from pypts.YamVIEW.verify_recipe import ( RecipeValidationError, validate_recipe_file, @@ -66,6 +66,6 @@ def test_wrapper_canonical_output_round_trips(tmp_path): path = tmp_path / "recipe.yml" path.write_text(VALID, encoding="utf-8") definition = parse_recipe_file(path).require_recipe() - canonical = dump_recipe(definition) + canonical = recipe_to_yaml(definition) valid, _ = validate_recipe_string_variable(canonical) assert valid From 1f14cd30ec83c37c44a87938a9611d54b8492aef Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 12:04:14 +0200 Subject: [PATCH 11/14] making yamview compatible with the latest recipe language changes --- docs/source/gui_architecture.rst | 42 +- docs/source/recipe_language_architecture.rst | 8 + src/pypts/YamVIEW/recipe_creator.py | 1318 ++++--------- src/pypts/YamVIEW/recipe_sequencer_setup.py | 876 ++++----- src/pypts/YamVIEW/recipe_step_setup.py | 1743 +++--------------- tests/unit_tests/test_recipe_creator.py | 124 -- tests/unit_tests/test_yamview_schema.py | 140 ++ tests/unit_tests/test_yamview_workflows.py | 235 +++ 8 files changed, 1425 insertions(+), 3061 deletions(-) delete mode 100644 tests/unit_tests/test_recipe_creator.py create mode 100644 tests/unit_tests/test_yamview_workflows.py diff --git a/docs/source/gui_architecture.rst b/docs/source/gui_architecture.rst index 88f5dc2..e25067b 100644 --- a/docs/source/gui_architecture.rst +++ b/docs/source/gui_architecture.rst @@ -145,11 +145,43 @@ The editor window contains: * a log console at the bottom * a watermark/empty-state screen shown when no recipe is open -The main editing widgets are still specific to ``YamVIEW``: - -* ``SequencerWidget`` for sequence/step manipulation -* ``ScintillaYamlEditor`` for YAML text editing and highlighting -* several recipe-editing dialogs in ``recipe_step_setup.py`` and related modules +The editing widgets are specific to ``YamVIEW``, with deliberately separate +responsibilities: + +* ``RecipeEditorMainMenu`` owns the working text, structural representation, + production-parser diagnostics, last-valid recovery state, and file I/O. +* ``SequencerWidget`` owns selection and emits add, edit, move, reorder, and + delete intents identified by sequence, stage, and step identity. It does not + define recipe fields or serialize YAML. +* ``Step_setup`` and its mapping rows render controls from the aggregate JSON + Schema. Pydantic remains the only owner of variants, required fields, strict + types, defaults, and field descriptions. +* ``ScintillaYamlEditor`` owns editable source text and diagnostic highlighting. + +Editor data flow +~~~~~~~~~~~~~~~~ + +The structured editing path is:: + + production Pydantic model -> aggregate JSON Schema -> schema form + -> working aggregate -> recipe_to_yaml() -> YAML editor + -> parse_recipe_text() -> status, diagnostics, and save state + +A structured edit is committed once its individual definition is valid. If a +cross-document or ordering rule then makes the aggregate invalid, the edit is +retained, diagnostics are shown, and Save and Save As are disabled until the +recipe is repaired or the last valid state is restored. Malformed JSON in an +individual structured field remains in its dialog and is not committed. + +Raw YAML editing is intentionally more permissive. Invalid text remains visible +and the first diagnostic span is highlighted. The sequencer stays available +for structurally valid recipes with semantic diagnostics, but is disabled when +the text cannot be represented safely as the aggregate model. + +Structured edits and saves use :func:`pypts.recipe_parser.recipe_to_yaml`. +Consequently, they are deterministic and preserve recipe meaning, but they do +not preserve comments, quoting, or source formatting. Raw text is never +silently lowercased or otherwise migrated. Relationship Between The Two GUIs --------------------------------- diff --git a/docs/source/recipe_language_architecture.rst b/docs/source/recipe_language_architecture.rst index 61081e8..ac7094f 100644 --- a/docs/source/recipe_language_architecture.rst +++ b/docs/source/recipe_language_architecture.rst @@ -208,6 +208,14 @@ not own supported step names or field rules. Whole-recipe validation always goes through the parser so semantic rules and YAML diagnostics are identical between YamVIEW, command-line tools, and runtime loading. +YamVIEW retains a locally valid structured edit even when whole-recipe +semantics fail. The parser diagnostics are displayed and persistence is +disabled until the edit is repaired or the last valid text is restored. +Schema-invalid raw YAML remains available in the text editor, while the +structured sequencer is disabled until an aggregate definition can be formed. +Structured edits and persistence call ``recipe_to_yaml`` and therefore replace +comments, quoting, and source layout with deterministic canonical formatting. + How the sequencer consumes the model ------------------------------------ diff --git a/src/pypts/YamVIEW/recipe_creator.py b/src/pypts/YamVIEW/recipe_creator.py index a81e8c7..44075fc 100644 --- a/src/pypts/YamVIEW/recipe_creator.py +++ b/src/pypts/YamVIEW/recipe_creator.py @@ -1,66 +1,86 @@ # SPDX-FileCopyrightText: 2025 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later -from pypts.YamVIEW.customGUIModules import ( - ScintillaYamlEditor, - WatermarkWidget, - HashableTreeItem, - RecipeCreatorApp -) +"""YamVIEW recipe editor shell and working-document owner.""" + +from __future__ import annotations + +import sys +import webbrowser +from pathlib import Path -import re -import io, uuid +import yaml +from pydantic import ValidationError +from PySide6.QtCore import QMargins, QSize, Qt +from PySide6.QtGui import ( + QAction, + QColor, + QFont, + QKeySequence, + QPixmap, + QShortcut, + QTextCursor, +) from PySide6.QtWidgets import ( QApplication, - QMainWindow, - QWidget, - QVBoxLayout, + QFileDialog, QHBoxLayout, + QLabel, + QMainWindow, + QSizePolicy, QStackedLayout, - QMessageBox, + QStyle, QTextEdit, - QLabel, - QWidgetAction, - QFileDialog, QToolBar, - QStyle, - QSizePolicy, + QVBoxLayout, + QWidget, + QWidgetAction, ) -from PySide6.QtGui import ( - QAction, - QTextCursor, - QPixmap + +from pypts.gui_theme import ( + detect_system_dark_mode, + get_theme_colors, + get_yamview_stylesheet, + install_system_theme_sync, ) -from PySide6.QtCore import QSize, QMargins, Qt -from ruamel.yaml import YAML -from ruamel.yaml.error import YAMLError -from datetime import datetime -import webbrowser -from pypts.YamVIEW.styles import * -from pypts.YamVIEW.verify_recipe import * -from pypts.recipe_parser import parse_recipe_text, recipe_to_yaml -import sys -from PySide6.QtGui import QColor, QTextCharFormat, QFont -from PySide6.QtCore import Qt -from PySide6.QtGui import QKeySequence, QShortcut -from pypts.YamVIEW.recipe_sequencer_setup import * -from pypts.gui_theme import detect_system_dark_mode, get_theme_colors, get_yamview_stylesheet, install_system_theme_sync +from pypts.recipe_language import Recipe as RecipeDefinition +from pypts.recipe_language import RecipeHeader, Sequence +from pypts.recipe_parser import ParseResult, parse_recipe_text, recipe_to_yaml +from pypts.YamVIEW.customGUIModules import ( + RecipeCreatorApp, + ScintillaYamlEditor, + WatermarkWidget, +) +from pypts.YamVIEW.recipe_sequencer_setup import SequencerWidget, _sequence_node +from pypts.YamVIEW.verify_recipe import format_diagnostic + +SEMANTIC_DIAGNOSTIC_CODES = { + "duplicate-sequence", + "unknown-main-sequence", + "unknown-sequence-reference", + "unequal-indexed-inputs", + "mixed-passthrough", + "missing-ssh-global", + "missing-ssh-credential", + "missing-ssh-connect", + "missing-ssh-close", +} class RecipeEditorMainMenu(QMainWindow): -# Initialization methods + """Own YamVIEW working text, validation state, recovery, and persistence.""" + def __init__(self): super().__init__() self.dark_mode = detect_system_dark_mode() - self.yaml_documents = [] self.temporary_recipe_contents = "" self.last_valid_recipe = "" - self.current_file_path = "" - self.item_to_line = {} - self.line_to_item = {} - self.yaml_parser = YAML() - self.data = None + self.current_file_path: str | None = None + self.current_recipe: RecipeDefinition | None = None + self.yaml_documents: list[dict] = [] + self.is_recipe_valid = False self.enable_recipe_verification = True + self._setting_text = False self.title = "YamVIEW 1.0.0" self.setWindowTitle(f"{self.title} recipe editor") self.setGeometry(200, 200, 1600, 1000) @@ -72,235 +92,180 @@ def __init__(self): self.setup_status_and_layouts() self.toggle_dark_mode_action.setChecked(self.dark_mode) self.toggle_dark_mode(self.dark_mode, log_change=False) - self._disconnect_system_theme_sync = install_system_theme_sync(QApplication.instance(), self._set_dark_mode) - - # Define the shortcut activation action - save_shortcut = QShortcut(QKeySequence("Ctrl+S"), self) - save_shortcut.activated.connect(self.on_save_clicked) - + self._disconnect_system_theme_sync = install_system_theme_sync( + QApplication.instance(), self._set_dark_mode + ) + self._update_actions() + shortcut = QShortcut(QKeySequence("Ctrl+S"), self) + shortcut.activated.connect(self.on_save_clicked) self.log("✅ Application started.") - def closeEvent(self, event): + def closeEvent(self, event) -> None: self._disconnect_system_theme_sync() super().closeEvent(event) - def setup_menu(self): + def setup_menu(self) -> None: menubar = self.menuBar() - # File menu self.file_menu = menubar.addMenu("File") - self.new_recipe_action = QAction("New Recipe", self) self.open_recipe_action = QAction("Open Recipe", self) self.close_recipe = QAction("Close Recipe", self) self.exit_action = QAction("Exit", self) - - self.file_menu.addAction(self.new_recipe_action) - self.file_menu.addAction(self.open_recipe_action) - self.file_menu.addAction(self.close_recipe) - self.file_menu.addAction(self.exit_action) - self.close_recipe.setEnabled(False) - + for action in ( + self.new_recipe_action, + self.open_recipe_action, + self.close_recipe, + self.exit_action, + ): + self.file_menu.addAction(action) self.new_recipe_action.triggered.connect(self.on_add_clicked) self.open_recipe_action.triggered.connect(self.open_recipe) self.close_recipe.triggered.connect(self.on_close_recipe_clicked) self.exit_action.triggered.connect(self.close) - # Edit menu self.edit_menu = menubar.addMenu("Edit") - self.save_action = QAction("Save Recipe", self) self.save_as_action = QAction("Save Recipe As", self) - self.edit_menu.addAction(self.save_action) self.edit_menu.addAction(self.save_as_action) - self.save_action.triggered.connect(self.on_save_clicked) self.save_as_action.triggered.connect(self.on_save_as_clicked) - # View menu self.view_menu = menubar.addMenu("View") self.toggle_dark_mode_action = QAction("Toggle Dark Mode", self) self.toggle_dark_mode_action.setCheckable(True) self.toggle_dark_mode_action.triggered.connect(self.toggle_dark_mode) self.view_menu.addAction(self.toggle_dark_mode_action) - # About menu self.about_menu = menubar.addMenu("About") - self.open_gitlab = QAction("Gitlab", self) - self.open_wiki = QAction("Wiki", self) - + self.open_gitlab = QAction("GitLab", self) + self.open_wiki = QAction("Documentation", self) self.about_menu.addAction(self.open_gitlab) self.about_menu.addAction(self.open_wiki) - - self.open_wiki.triggered.connect(self.on_open_wiki_clicked) self.open_gitlab.triggered.connect(self.on_open_gitlab_clicked) + self.open_wiki.triggered.connect(self.on_open_wiki_clicked) - # Development menu - self.dev_menu = menubar.addMenu("Development") - for i in range(1, 5): - action = QAction(f"Dev Tool {i}", self) - action.triggered.connect(getattr(self, f"on_dev_{i}_clicked")) - self.dev_menu.addAction(action) - self.dev_menu.addAction(action) - - def setup_central_widget(self): + def setup_central_widget(self) -> None: self.central_widget = QWidget() self.central_widget.setObjectName("yamRoot") self.setCentralWidget(self.central_widget) self.main_layout = QVBoxLayout(self.central_widget) - def setup_toolbar(self): + def setup_toolbar(self) -> None: self.toolbar = QToolBar() self.toolbar.setIconSize(QSize(28, 28)) - self.toolbar.setToolButtonStyle(Qt.ToolButtonStyle.ToolButtonIconOnly) - - self.action_add = QAction(self.style().standardIcon(QStyle.SP_FileDialogNewFolder), "Create recipe from the template", self) - self.action_save = QAction(self.style().standardIcon(QStyle.SP_DialogSaveButton), "Save", self) - self.action_save_as = QAction(self.style().standardIcon(QStyle.SP_DialogSaveButton), "Save as", self) - self.action_restore_recipe = QAction(self.style().standardIcon(QStyle.SP_BrowserReload), - "Restore last working recipe state", self) - - self.toolbar.addAction(self.action_add) - self.toolbar.addAction(self.action_save) - self.toolbar.addAction(self.action_save_as) - self.toolbar.addAction(self.action_restore_recipe) - + self.action_add = QAction( + self.style().standardIcon(QStyle.SP_FileDialogNewFolder), + "Create recipe from template", + self, + ) + self.action_save = QAction( + self.style().standardIcon(QStyle.SP_DialogSaveButton), "Save", self + ) + self.action_save_as = QAction( + self.style().standardIcon(QStyle.SP_DialogSaveButton), "Save as", self + ) + self.action_restore_recipe = QAction( + self.style().standardIcon(QStyle.SP_BrowserReload), + "Restore last valid recipe state", + self, + ) + for action in ( + self.action_add, + self.action_save, + self.action_save_as, + self.action_restore_recipe, + ): + self.toolbar.addAction(action) self.action_add.triggered.connect(self.on_add_clicked) self.action_save.triggered.connect(self.on_save_clicked) self.action_save_as.triggered.connect(self.on_save_as_clicked) self.action_restore_recipe.triggered.connect(self.on_action_restore_recipe_clicked) - - self.save_as_action.setEnabled(False) - self.save_action.setEnabled(False) - self.action_save.setEnabled(False) - self.action_save_as.setEnabled(False) - spacer = QWidget() spacer.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred) spacer_action = QWidgetAction(self) spacer_action.setDefaultWidget(spacer) self.toolbar.addAction(spacer_action) - icon_label = QLabel() icon_label.setPixmap(QPixmap("../images/YamVIEW_cookie.png")) icon_action = QWidgetAction(self) icon_action.setDefaultWidget(icon_label) self.toolbar.addAction(icon_action) - - def setup_tree_and_yaml(self): - # YAML Viewer (custom ScintillaYamlEditor) + def setup_tree_and_yaml(self) -> None: self.yaml_viewer = ScintillaYamlEditor(self) self.yaml_viewer.setReadOnly(False) self.yaml_viewer.textChanged.connect(self.on_yamlview_item_changed) - self.yaml_viewer.cursorPositionChanged.connect(self.on_yaml_cursor_changed) font = QFont("Fira Code", 11) font.setStyleHint(QFont.StyleHint.Monospace) font.setFixedPitch(True) self.yaml_viewer.setFont(font) - - #Sequencer - self.sequencer = SequencerWidget(yaml_viewer=self.yaml_viewer) + self.sequencer = SequencerWidget(self.yaml_viewer) self.sequencer.yaml_update_callback = self.on_sequencer_updated - - # Horizontal container for tree + yaml viewer self.tree_and_yaml_widget = QWidget() - self.tree_and_yaml_hlayout = QHBoxLayout(self.tree_and_yaml_widget) - self.tree_and_yaml_hlayout.addWidget(self.sequencer) - self.tree_and_yaml_hlayout.addWidget(self.yaml_viewer) + layout = QHBoxLayout(self.tree_and_yaml_widget) + layout.addWidget(self.sequencer) + layout.addWidget(self.yaml_viewer) - def setup_status_and_layouts(self): - # Recipe status bar + def setup_status_and_layouts(self) -> None: self.recipeStatus = QTextEdit() self.recipeStatus.setObjectName("recipeStatus") self.recipeStatus.setReadOnly(True) self.recipeStatus.setFixedHeight(30) self.recipeStatus.setViewportMargins(QMargins(5, 0, 0, 0)) - - # Container for status + tree+yaml - self.tree_status_container = QWidget() - self.tree_status_layout = QVBoxLayout(self.tree_status_container) - self.tree_status_layout.setContentsMargins(0, 0, 0, 0) - self.tree_status_layout.setSpacing(0) - self.tree_status_layout.addWidget(self.recipeStatus) - self.tree_status_layout.addWidget(self.tree_and_yaml_widget) - - # Container for toolbar + status + main content - self.tree_and_yaml_container = QWidget() - self.tree_and_yaml_layout = QVBoxLayout(self.tree_and_yaml_container) - self.tree_and_yaml_layout.setContentsMargins(0, 0, 0, 0) - self.tree_and_yaml_layout.setSpacing(0) - self.tree_and_yaml_layout.addWidget(self.toolbar) - self.tree_and_yaml_layout.addWidget(self.tree_status_container) - - # Watermark widget (your logo) - self.watermark_widget = WatermarkWidget("../images/CERN_Logo.png") # Replace with your logo - - # Stacked layout to switch between watermark and main editor + content = QWidget() + content_layout = QVBoxLayout(content) + content_layout.setContentsMargins(0, 0, 0, 0) + content_layout.setSpacing(0) + content_layout.addWidget(self.recipeStatus) + content_layout.addWidget(self.tree_and_yaml_widget) + editor = QWidget() + editor_layout = QVBoxLayout(editor) + editor_layout.setContentsMargins(0, 0, 0, 0) + editor_layout.setSpacing(0) + editor_layout.addWidget(self.toolbar) + editor_layout.addWidget(content) + self.watermark_widget = WatermarkWidget("../images/CERN_Logo.png") self.stacked_layout = QStackedLayout() - self.stacked_layout.addWidget(self.watermark_widget) # index 0 - self.stacked_layout.addWidget(self.tree_and_yaml_container) # index 1 - - # Log console below everything + self.stacked_layout.addWidget(self.watermark_widget) + self.stacked_layout.addWidget(editor) self.log_console = QTextEdit() self.log_console.setObjectName("yamLogConsole") self.log_console.setReadOnly(True) self.log_console.setFixedHeight(200) - - # Add layouts to main layout self.main_layout.addLayout(self.stacked_layout) self.main_layout.addWidget(self.log_console) -# Helper GUI methods - colouring, viewing - - def toggle_dark_mode(self, enabled, log_change=True): + def toggle_dark_mode(self, enabled, log_change=True) -> None: self.dark_mode = bool(enabled) self.setStyleSheet(get_yamview_stylesheet(self.dark_mode)) self.yaml_viewer.set_dark_mode(self.dark_mode) self.sequencer.set_dark(self.dark_mode) - colors = get_theme_colors(self.dark_mode) self.recipeStatus.document().setDefaultStyleSheet( f"body {{ color: {colors['header_text']}; }}" ) - if log_change: - if self.dark_mode: - self.log("🌙 Dark Mode enabled.") - else: - self.log("☀️ Light Mode restored.") + self.log("🌙 Dark Mode enabled." if self.dark_mode else "☀️ Light Mode restored.") - def _set_dark_mode(self, enabled): + def _set_dark_mode(self, enabled) -> None: self.toggle_dark_mode_action.setChecked(enabled) self.toggle_dark_mode(enabled) - def highlight_line(self, line_num): - cursor = self.yaml_viewer.textCursor() - - # Clear any existing selections - cursor.clearSelection() - # Move to the specified line - cursor.movePosition(QTextCursor.MoveOperation.Start) - for _ in range(line_num): - cursor.movePosition(QTextCursor.MoveOperation.Down) - - # Select the entire line - cursor.select(QTextCursor.SelectionType.LineUnderCursor) + def set_recipe_status(self, message: str, color: str = "#333") -> None: + escaped = message.replace("&", "&").replace("<", "<").replace(">", ">") + self.recipeStatus.setHtml(f'{escaped}') - # Apply highlighting - format = QTextCharFormat() + def show_recipe_ok(self, message: str = "✅ Recipe is valid") -> None: + self.set_recipe_status(message, "green") - self.enable_recipe_verification = False - cursor.mergeCharFormat(format) - self.enable_recipe_verification = True + def show_recipe_error(self, message: str) -> None: + self.set_recipe_status(f"❌ {message}", "red") - # Scroll and set cursor position for visibility - self.yaml_viewer.setCursorPosition(line_num, 0) - self.yaml_viewer.ensureLineVisible(line_num) - pass + def show_recipe_info(self, message: str) -> None: + self.set_recipe_status(f"ℹ️ {message}", "gray") - def highlight_diagnostic(self, diagnostic): - """Highlight the exact parser span for one editor diagnostic.""" + def highlight_diagnostic(self, diagnostic) -> None: if diagnostic.span is None: return selection = QTextEdit.ExtraSelection() @@ -316,814 +281,289 @@ def highlight_diagnostic(self, diagnostic): self.yaml_viewer.setTextCursor(cursor) self.yaml_viewer.ensureCursorVisible() - def update_yaml_viewer(self): - self.temporary_recipe_contents = self.sanitize_booleans(self.temporary_recipe_contents) - self.yaml_viewer.setText(self.temporary_recipe_contents) - - def update_yaml_treeview(self): - """Load YAML documents and populate the sequencer with folders for setup, main, and teardown steps.""" + def _set_text(self, text: str) -> None: + self._setting_text = True + self.temporary_recipe_contents = text + self.yaml_viewer.setText(text) + self._setting_text = False + + def update_yaml_viewer(self) -> None: + """Display working text exactly; YamVIEW performs no silent normalization.""" + self._set_text(self.temporary_recipe_contents) + + @staticmethod + def _structural_recipe(text: str, parsed: ParseResult) -> RecipeDefinition | None: + if parsed.is_valid: + return parsed.recipe + if not parsed.diagnostics or any( + item.code not in SEMANTIC_DIAGNOSTIC_CODES for item in parsed.diagnostics + ): + return None try: - self.yaml_documents = list(self.yaml_parser.load_all(self.temporary_recipe_contents)) - if not self.yaml_documents: - self.log("⚠️ No YAML documents found.") - self.sequencer.set_yaml_data([]) - return False - - sequencer_steps = [] - - # First document → Preamble - first_doc = self.yaml_documents[0] - sequencer_steps.append({ + documents = list(yaml.safe_load_all(text)) + return RecipeDefinition( + header=RecipeHeader.model_validate(documents[0]), + sequences=[Sequence.model_validate(item) for item in documents[1:]], + ) + except (IndexError, TypeError, ValidationError, yaml.YAMLError): + return None + + def _populate_sequencer(self, recipe: RecipeDefinition) -> None: + header = recipe.header.model_dump(mode="python", by_alias=True, exclude_none=True) + nodes = [ + { "step_name": "Preamble", - "description": str(first_doc), "steptype": "preamble", - "_node": first_doc, - "_id": str(uuid.uuid4()) - }) - - # Process remaining documents - for doc_index, doc in enumerate(self.yaml_documents[1:], start=1): - sequence_id = str(uuid.uuid4()) - sequence_name = doc.get("sequence_name", f"Sequence {doc_index}") - sequence_block = { - "step_name": f"Sequence: {sequence_name}", - "description": doc.get("description", ""), - "steptype": "sequence_folder", - "children": [], - "_node": doc, - "_doc_index": doc_index, - "_sequence_id": sequence_id - } - - def build_folder(folder_name, folder_type, steps_list): - """Create a folder dict with child steps.""" - folder = { - "step_name": folder_name, - "steptype": folder_type, - "children": [], - "_node": steps_list, # original YAML list - "_expanded": False, - "_sequence_id": sequence_id - } - for s in steps_list: - if isinstance(s, dict) and s.get("step_name"): - folder["children"].append({ - "step_name": s.get("step_name", "Unnamed Step"), - "steptype": s.get("steptype", "unknown"), - "_node": s, - "_parent": folder_type, - "_id": s.get("_id", str(uuid.uuid4())), - "_sequence_id": sequence_id - }) - return folder - - # Build folders in chronological order: setup → main → teardown - folders = [ - build_folder("Setup Steps", "setup_folder", doc.get("setup_steps", [])), - build_folder("Main Steps", "main_folder", doc.get("steps", [])), - build_folder("Teardown Steps", "teardown_folder", doc.get("teardown_steps", [])), - ] - - sequence_block["children"].extend(folders) - sequencer_steps.append(sequence_block) - - self.sequencer.set_yaml_data(sequencer_steps) - self.sequencer.receive_globals(self.yaml_documents[0].get("globals", {})) - self.sequencer.receive_locals(self.yaml_documents[1].get("locals", {})) - self.log(f"✅ Sequencer received {len(sequencer_steps)} steps including preamble/setup.") - return True - - except Exception as e: - self.log(f"❌ Unexpected error during TreeView update: {e}") - return False - - - def on_sequencer_updated(self, steps): - """Update the YAML document with the current sequencer order.""" - if len(self.yaml_documents) < 2: - return - - if self.sequencer.updated_preamble_globals: - yaml_globals = self.yaml_documents[0].get("globals", {}) - yaml_globals.update(self.sequencer.updated_preamble_globals) - self.yaml_documents[0]["globals"] = yaml_globals - self.sequencer.receive_globals(yaml_globals) - self.sequencer.updated_preamble_globals = {} - - if self.sequencer.updated_sequence_locals: - yaml_locals = self.yaml_documents[1].get("locals", {}) - yaml_locals.update(self.sequencer.updated_sequence_locals) - self.yaml_documents[1]["locals"] = yaml_locals - self.sequencer.receive_locals(yaml_locals) - self.sequencer.updated_sequence_locals = {} - - if self.sequencer.new_sequence_request: - seq = self.sequencer.new_sequence_request - - new_doc = { - "sequence_name": seq.get("sequence_name", "New Sequence"), - "description": seq.get("description", ""), - "parameters": seq.get("parameters", {}), - "locals": seq.get("locals", {}), - "outputs": seq.get("outputs", {}), - "setup_steps": [], - "steps": [], - "teardown_steps": [] + "_node": header, + "_id": "preamble", } - # Add new document - self.yaml_documents.append(new_doc) - - # attach index inside sequencer structure - seq["_doc_index"] = len(self.yaml_documents) - 1 - - # clear request - self.sequencer.new_sequence_request = None - + ] + for index, sequence in enumerate(recipe.sequences, start=1): + document = sequence.model_dump(mode="python", by_alias=True, exclude_none=True) + nodes.append(_sequence_node(document, f"sequence:{index}")) + self.sequencer.set_yaml_data(nodes) + + def _update_actions(self) -> None: + has_recipe = bool(self.temporary_recipe_contents) + can_save = has_recipe and self.is_recipe_valid + self.close_recipe.setEnabled(has_recipe) + self.save_as_action.setEnabled(can_save) + self.action_save_as.setEnabled(can_save) + self.save_action.setEnabled(can_save and bool(self.current_file_path)) + self.action_save.setEnabled(can_save and bool(self.current_file_path)) + self.action_restore_recipe.setEnabled( + bool(self.last_valid_recipe) + and self.temporary_recipe_contents != self.last_valid_recipe + ) - yaml_doc_index = 1 # doc 0 is preamble + def _validate_working_text(self, *, rebuild_sequencer: bool) -> ParseResult: + parsed = parse_recipe_text(self.temporary_recipe_contents, "") + structural = self._structural_recipe(self.temporary_recipe_contents, parsed) + self.current_recipe = structural + self.is_recipe_valid = parsed.is_valid + self.yaml_viewer.setExtraSelections([]) + if parsed.is_valid: + self.last_valid_recipe = self.temporary_recipe_contents + self.show_recipe_ok() + else: + message = format_diagnostic(parsed.diagnostics[0]) if parsed.diagnostics else "Invalid recipe" + self.show_recipe_error(message) + for diagnostic in parsed.diagnostics: + self.log(format_diagnostic(diagnostic)) + if parsed.diagnostics: + self.highlight_diagnostic(parsed.diagnostics[0]) + if rebuild_sequencer: + if structural is None: + self.sequencer.clear() + self.sequencer.setEnabled(False) + else: + self.sequencer.setEnabled(True) + self._populate_sequencer(structural) + self._update_actions() + return parsed - for seq_step in steps: - if seq_step.get("steptype") != "sequence_folder": + def update_yaml_treeview(self) -> bool: + parsed = parse_recipe_text(self.temporary_recipe_contents, "") + structural = self._structural_recipe(self.temporary_recipe_contents, parsed) + if structural is None: + self.sequencer.clear() + self.sequencer.setEnabled(False) + return False + self.sequencer.setEnabled(True) + self._populate_sequencer(structural) + return True + + def _recipe_from_sequencer(self, nodes) -> RecipeDefinition: + header_node = next(item for item in nodes if item.get("steptype") == "preamble") + sequence_documents = [] + for sequence_node in nodes: + if sequence_node.get("steptype") != "sequence_folder": continue - - if yaml_doc_index >= len(self.yaml_documents): - break - - doc = self.yaml_documents[yaml_doc_index] - - for folder in seq_step.get("children", []): - folder_type = folder.get("steptype") - children_nodes = [child["_node"] for child in folder.get("children", [])] - - if folder_type == "setup_folder": - doc["setup_steps"] = children_nodes - elif folder_type == "main_folder": - doc["steps"] = children_nodes - elif folder_type == "teardown_folder": - doc["teardown_steps"] = children_nodes - - yaml_doc_index += 1 - - # Dump YAML in order: preamble → setup → main → teardown - from ruamel.yaml import YAML - import io - - yaml = YAML() - yaml.indent(mapping=2, sequence=4, offset=2) - stream = io.StringIO() - yaml.dump_all(self.yaml_documents, stream) - new_yaml = stream.getvalue() - - self.yaml_viewer.setText(new_yaml) - self.temporary_recipe_contents = new_yaml - self.log("✅ Sequencer order updated and YAML synced.") - - def set_recipe_status(self, message: str, color: str = "#333"): - """Sets the status message and text color in the top recipe status field.""" - escaped_message = ( - message.replace("&", "&") - .replace("<", "<") - .replace(">", ">") + document = dict(sequence_node["_node"]) + for folder in sequence_node["children"]: + values = [child["_node"] for child in folder["children"]] + if folder["steptype"] == "setup_folder": + document["setup_steps"] = values + elif folder["steptype"] == "main_folder": + document["steps"] = values + elif folder["steptype"] == "teardown_folder": + document["teardown_steps"] = values + sequence_documents.append(document) + return RecipeDefinition.model_validate( + {"header": header_node["_node"], "sequences": sequence_documents} ) - self.recipeStatus.setHtml(f'{escaped_message}') - def show_recipe_ok(self, message: str = "✅ Recipe is valid"): - self.set_recipe_status(message, color="green") - - def show_recipe_error(self, message: str): - self.set_recipe_status(f"❌ {message}", color="red") - - def show_recipe_info(self, message: str): - self.set_recipe_status(f"ℹ️ {message}", color="gray") - - def on_yaml_cursor_changed(self): + def on_sequencer_updated(self, nodes) -> bool: + """Commit a structurally valid GUI edit and re-run production semantics.""" try: - cursor = self.yaml_viewer.textCursor() - current_line = cursor.blockNumber() + recipe = self._recipe_from_sequencer(nodes) + except (StopIteration, ValidationError) as error: + self.show_recipe_error(str(error)) + if self.current_recipe is not None: + self._populate_sequencer(self.current_recipe) + self._update_actions() + return False + canonical = recipe_to_yaml(recipe) + self.current_recipe = recipe + self._set_text(canonical) + self._validate_working_text(rebuild_sequencer=False) + self.log("✏️ Structured edit committed; YAML formatting was normalized.") + self._mark_unsaved() + return True + + def on_yamlview_item_changed(self) -> None: + if self._setting_text or not self.enable_recipe_verification: + return + self.temporary_recipe_contents = self.yaml_viewer.toPlainText() + self._validate_working_text(rebuild_sequencer=True) + self._mark_unsaved() - wrapper = self.line_to_item.get(current_line) - item = wrapper.item if isinstance(wrapper, HashableTreeItem) else wrapper + def validate_temporary_recipe_contents(self) -> tuple[bool, str]: + parsed = self._validate_working_text(rebuild_sequencer=False) + description = "\n".join(format_diagnostic(item) for item in parsed.diagnostics) + return parsed.is_valid, description or "Validation passed for the recipe." - if item: - self.tree.setCurrentItem(item) - self.tree.scrollToItem(item) - # self.log(f"🧭 Cursor at line {current_line}, highlighting: {item.text(0)}") - else: - # self.log(f"🧭 Cursor at line {current_line}, no matching tree item found.") - pass - except Exception as e: - self.log(f"❌ Error in on_yaml_cursor_changed: {e}") + def validate_recipe(self) -> bool: + valid, _ = self.validate_temporary_recipe_contents() + return valid - # Methods related to handling GUI actions - def on_action_restore_recipe_clicked(self): - if self.last_valid_recipe == "": - self.log("️⚠️ ️Unable to restore, no working version in the history") + def _mark_unsaved(self) -> None: + if not self.temporary_recipe_contents: return - self.temporary_recipe_contents = self.last_valid_recipe - self.update_yaml_viewer() - #self.collapse_inside_steps() - - def on_save_as_clicked(self): - try: - # Validate recipe - validation_result, description = self.validate_temporary_recipe_contents() - if not validation_result: - self.log("⚠️ Canonical save is blocked until the recipe validates as version 2.0.0.") - self.log(description) - return + filename = Path(self.current_file_path).name if self.current_file_path else "unnamed recipe" + self.setWindowTitle(f"Recipe Editor - {filename} *unsaved changes*") - except Exception as e: - self.log(f"❌ Recipe validation failed: {e}") + def on_action_restore_recipe_clicked(self) -> None: + if not self.last_valid_recipe: + self.log("⚠️ Unable to restore: no valid version is available.") + return + self._set_text(self.last_valid_recipe) + self._validate_working_text(rebuild_sequencer=True) + self.log("↩️ Restored the last valid recipe state.") - # Open the file dialog for the user to choose the save location - new_path, _ = QFileDialog.getSaveFileName( + def _canonical_text(self) -> str | None: + parsed = parse_recipe_text(self.temporary_recipe_contents, "") + if not parsed.is_valid: + self._validate_working_text(rebuild_sequencer=False) + self.log("⚠️ Save is blocked until the recipe validates.") + return None + return recipe_to_yaml(parsed.require_recipe()) + + def _write_recipe(self, path: Path) -> bool: + text = self._canonical_text() + if text is None: + return False + try: + path.write_text(text, encoding="utf-8") + except OSError as error: + self.log(f"❌ Save failed: {error}") + return False + self.current_file_path = str(path) + self._set_text(text) + self._validate_working_text(rebuild_sequencer=True) + self.setWindowTitle(f"Recipe Editor - {path.name}") + self.log(f"💾 Saved canonical YAML to {path}") + return True + + def on_save_as_clicked(self) -> None: + if not self.is_recipe_valid: + self.log("⚠️ Save As is blocked until the recipe validates.") + return + chosen, _ = QFileDialog.getSaveFileName( self, "Save As", - self.current_file_path or "", # start dir - "YAML Files (*.yaml *.yml);;All Files (*)" + self.current_file_path or "", + "YAML Files (*.yaml *.yml);;All Files (*)", ) - - if not new_path: - self.log("⚠️ Save aborted, no YAML file selected.") + if not chosen: + return + path = Path(chosen) + if path.suffix.lower() not in {".yaml", ".yml"}: + path = path.with_suffix(".yml") + self._write_recipe(path) + + def on_save_clicked(self) -> None: + if not self.is_recipe_valid: + self.log("⚠️ Save is blocked until the recipe validates.") return - - try: - # Extract data from the text view - try: - # data = self.extract_treeView_to_data() - data = recipe_to_yaml( - parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() - ) - except Exception as e: - raise RuntimeError(f"Failed to extract data from the text editor: {e}") - try: - base, _ = os.path.splitext(new_path) - new_path_fixed = f"{base}.yml" - with open(new_path_fixed, 'w') as f: - # yaml.dump_all(data, f, sort_keys=False) - f.write(data) - except Exception as e: - raise IOError(f"Failed to save YAML file '{new_path_fixed}': {e}") - - # Update UI state and log success - self.current_file_path = new_path_fixed - self.save_action.setEnabled(True) - self.action_save.setEnabled(True) - self.log(f"💾 Saved to {new_path_fixed}") - self.setWindowTitle(f"Recipe Editor - {os.path.basename(new_path_fixed)}") - - try: - self.load_yaml_recipe(new_path_fixed) - except Exception as e: - self.log(f"⚠️ Saved but failed to reload: {e}") - except Exception as e: - error_message = f"❌ Save failed: {e}" - self.log(error_message) - - def on_save_clicked(self): if not self.current_file_path: - self.on_save_as_clicked() # Fallback + self.on_save_as_clicked() return + self._write_recipe(Path(self.current_file_path)) - try: - # Validate recipe - validation_result, description = self.validate_temporary_recipe_contents() - if not validation_result: - self.log("⚠️ Canonical save is blocked until the recipe validates as version 2.0.0.") - self.log(description) - return - - if not self.current_file_path: - self.log("⚠️ Save aborted, no YAML file selected.") - return - - # Extract data from text view - try: - # data = self.extract_treeView_to_data() - data = recipe_to_yaml( - parse_recipe_text(self.temporary_recipe_contents, "").require_recipe() - ) - except Exception as e: - raise RuntimeError(f"Failed to extract data from the text view: {e}") - - # Construct filename and write data - try: - base, ext = os.path.splitext(self.current_file_path) - new_path = f"{base}{ext}" - with open(new_path, 'w') as f: - # yaml.dump_all(data, f, sort_keys=False) - f.write(data) - except Exception as e: - raise IOError(f"Failed to save YAML file '{new_path}': {e}") - - # Log success and reload - self.log(f"💾 Saved to {new_path}") - self.setWindowTitle(f"Recipe Editor - {os.path.basename(new_path)}") - - try: - self.load_yaml_recipe(new_path) - except Exception as e: - self.log(f"⚠️ Saved but failed to reload: {e}") - except Exception as e: - self.log(f"❌ Save failed: {e}") - - def on_add_clicked(self): - generator_pop_up = RecipeCreatorApp() - result = generator_pop_up.open_creator_dialog(self.dark_mode) - if result == None: + def on_add_clicked(self) -> None: + generator = RecipeCreatorApp() + if generator.open_creator_dialog(self.dark_mode) is None: return - - filename = os.path.basename(self.current_file_path) - if filename == "": - self.setWindowTitle(f"Recipe Editor - *unnamed recipe* *unsaved changes*") - else: - self.setWindowTitle(f"Recipe Editor - {filename} *unsaved changes*") - self.close_recipe.setEnabled(True) - self.action_restore_recipe.setEnabled(True) - yaml_string = generator_pop_up.get_generated_recipe() - self.temporary_recipe_contents = yaml_string - self.update_yaml_viewer() - - # Mark as unsaved self.current_file_path = None - self.stacked_layout.setCurrentIndex(1) - - #self.collapse_inside_steps() - - self.action_save.setEnabled(False) - self.action_save_as.setEnabled(True) - self.save_action.setEnabled(False) - self.save_as_action.setEnabled(True) - - def on_open_wiki_clicked(self): - url = "https://acc-py.web.cern.ch/gitlab/pts/framework/pypts/docs/master/" - webbrowser.open(url) - - def on_dev_1_clicked(self): - self.log("clicked Dev1 tool (no action implemented)") - - def on_dev_2_clicked(self): - self.log("clicked Dev2 tool (no action implemented)") - - def on_dev_3_clicked(self): - self.log("clicked Dev3 tool (no action implemented)") - - def on_dev_4_clicked(self): - self.log("clicked Dev4 tool (no action implemented)") - - def on_open_gitlab_clicked(self): - url = "https://gitlab.cern.ch/pts/framework/pypts" - webbrowser.open(url) - - def on_close_recipe_clicked(self): - #self.tree.clear() - self.sequencer.clear() - self.stacked_layout.setCurrentIndex(0) # Show logo - self.close_recipe.setEnabled(False) # 🔒 Disable it - gray out - self.action_save.setEnabled(False) - self.action_save_as.setEnabled(False) - self.save_as_action.setEnabled(False) - self.save_action.setEnabled(False) - - self.yaml_documents = [] - self.temporary_recipe_contents = "" - self.last_valid_recipe = "" - self.current_file_path = "" - self.setWindowTitle(f"{self.title} recipe editor") - self.log("️ℹ️ Recipe view cleared.") self.reset_recovery_history() + self._set_text(generator.get_generated_recipe()) + self.stacked_layout.setCurrentIndex(1) + self._validate_working_text(rebuild_sequencer=True) + self._mark_unsaved() - def on_create_recipe_clicked(self): - self.log("️ℹ️ Todo: 0.2") - - def on_tree_item_clicked(self, item, column): - key = HashableTreeItem(item) - if key in self.item_to_line: - line = self.item_to_line[key] - self.highlight_line(line) - else: - pass - - def on_treeview_item_changed(self, item, column): - if self.enable_recipe_verification == True: - line_info = "" - line_number = self.item_to_line.get(HashableTreeItem(item)) - if line_number is not None: - line_info = f" (line {line_number + 1})" # +1 for human-readable line number - - if item.text(1) == "": - self.log(f"✏️ Cleared a field...{line_info}") - else: - self.log(f"✏️ Recipe in Tree editor updated") - - self.extract_treeView_to_data() - self.temporary_recipe_contents = self.sanitize_booleans(self.temporary_recipe_contents) - self.update_yaml_viewer() - result, description = self.validate_yaml_documents() - if result == True: - self.show_recipe_ok() - else: - self.show_recipe_error("Recipe in Tree Edit View is invalid!") - self.log(description) - try: - filename = "" - filename = os.path.basename(self.current_file_path) - except Exception as e: - pass - if filename == "": - self.setWindowTitle(f"Recipe Editor - *unnamed recipe* *unsaved changes*") - else: - self.setWindowTitle(f"Recipe Editor - {filename} *unsaved changes*") - pass - - def on_yamlview_item_changed(self): - if self.enable_recipe_verification == True: - try: - self.temporary_recipe_contents = self.yaml_viewer.toPlainText() - validation_result, description = self.validate_temporary_recipe_contents() - except Exception as e: - self.log(f"❌ Unable to parse YAML contents") - return - try: - if (validation_result == True): - self.update_yaml_treeview() - self.show_recipe_ok() - self.log("✏️️ Recipe in YAML editor updated") - else: - self.show_recipe_error("Recipe in Text Edit View is invalid!") - self.log(f"❌ YAML edit failure, the YAML format got corrupted!") - - except Exception as e: - self.log(f"❌ Could not update the view. {e}") - try: - filename = os.path.basename(self.current_file_path) - if filename == "": - self.setWindowTitle(f"Recipe Editor - *unnamed recipe* *unsaved changes*") - else: - self.setWindowTitle(f"Recipe Editor - {filename} *unsaved changes*") - except Exception as e: - pass - pass - - def keyPressEvent(self, event): - if event.key() == Qt.Key_Delete: - self.delete_selected_items() - else: - super().keyPressEvent(event) - -# Recipe parsing and processing - def sanitize_booleans(self, yaml_str: str) -> str: - sanitized_lines = [] - - for line in yaml_str.splitlines(): - # Convert the line to lowercase - line = line.lower() - - # Remove quotes around 'true' or 'false' - line = re.sub(r"(['\"])\s*(true|false)\s*\1", r"\2", line) - - sanitized_lines.append(line) - - return "\n".join(sanitized_lines) - - def validate_recipe(self): - try: - if (validate_recipe_filepath(self.current_file_path)): - self.log("✅ Recipe file validated successfully.") - self.show_recipe_ok() - else: - self.log("❌ Recipe file failed the validation!") - self.show_recipe_error("❌ Recipe file is invalid!") - return False - except Exception as e: - self.sequencer.blockSignals(False) # Safety catch - self.log(f"❌ Expception while validating the recipe, recipe might be corrupted.") - return False - - def validate_yaml_documents(self): - self.extract_treeView_to_data() - validation_result = self.validate_temporary_recipe_contents() - if validation_result == True: - self.last_valid_recipe = self.temporary_recipe_contents - self.action_restore_recipe.setEnabled(True) - return validation_result - - def validate_temporary_recipe_contents(self): - parsed = parse_recipe_text(self.temporary_recipe_contents, "") - result, description = validate_recipe_string_variable(self.temporary_recipe_contents) - if result == True: - self.last_valid_recipe = self.temporary_recipe_contents - elif parsed.diagnostics and parsed.diagnostics[0].span is not None: - self.highlight_diagnostic(parsed.diagnostics[0]) - return result, description - - def open_recipe(self): - if getattr(self, "current_file_path", None): + def open_recipe(self) -> None: + if self.current_file_path: file_path = self.current_file_path - - else: - file_path, _ = QFileDialog.getOpenFileName(self, "Open recipe file", "", "YAML Files (*.yml *.yaml)") - self.load_yaml_recipe(file_path) - self.save_as_action.setEnabled(True) - self.save_action.setEnabled(True) - self.action_save.setEnabled(True) - self.action_save_as.setEnabled(True) - #self.collapse_inside_steps() - - filename = os.path.basename(file_path) - if filename == "": - return else: - self.setWindowTitle(f"Recipe Editor - {filename}") - - def reset_recovery_history(self): - self.last_valid_recipe = "" - self.action_restore_recipe.setEnabled(False) - - def load_yaml_recipe(self, file_path): + file_path, _ = QFileDialog.getOpenFileName( + self, "Open recipe file", "", "YAML Files (*.yml *.yaml)" + ) if file_path: - try: - with open(file_path, 'r') as f: - raw_text = f.read() - self.temporary_recipe_contents = raw_text - self.current_file_path = file_path - - - self.yaml_parser.preserve_quotes = True - - parsed = parse_recipe_text(raw_text, file_path) - self.enable_recipe_verification = False - self.update_yaml_viewer() - self.enable_recipe_verification = True - - try: - validation_result = parsed.is_valid - description = "\n".join(format_diagnostic(item) for item in parsed.diagnostics) - if validation_result: - self.update_yaml_treeview() - self.last_valid_recipe = raw_text - self.show_recipe_ok("✅ Recipe is valid") - else: - self.yaml_documents = [] - self.sequencer.set_yaml_data([]) - self.show_recipe_error("Opened recipe is invalid!") - self.log(description) - if parsed.diagnostics and parsed.diagnostics[0].span is not None: - self.highlight_diagnostic(parsed.diagnostics[0]) - except Exception as e: - self.log(f"❌ YAML verification failed, {e}") - - self.log(f"Loaded recipe text from: {file_path}") - - - self.stacked_layout.setCurrentIndex(1) - self.close_recipe.setEnabled(True) # 🔒 Enable it - - except YAMLError as e: - #self.tree.blockSignals(False) # Safety catch - self.log(f"❌ YAML parse error: {e}") - #self.collapse_inside_steps() - QApplication.processEvents() - - def on_steps_reordered(self, parent, start, end, destination, row): - new_order = [] - for i in range(self.list_widget.count()): - item = self.list_widget.item(i) - block = self.list_widget.itemWidget(item) - step = block.step_data - step["step_name"] = block.step_name - new_order.append(step) - - self.steps = new_order - if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) - - - def extract_treeView_to_data(self): - documents = [] - for i in range(self.tree.topLevelItemCount()): - doc_item = self.tree.topLevelItem(i) - data = self.extract_item_data(doc_item) - - doc_item = self.strip_star_prefix(doc_item) - data = self.strip_star_prefix(data) - - # Sanitize empty strings to empty dicts for specific keys - data = self.sanitize_empty_fields(data) - - documents.append(data) - - buffer = io.StringIO() - yaml.dump_all(documents, buffer, sort_keys=False) - yaml_string = buffer.getvalue() - buffer.close() - self.temporary_recipe_contents = yaml_string - self.temporary_recipe_contents = self.sanitize_booleans(self.temporary_recipe_contents) - return documents - - def extract_item_data(self, item): - if item.childCount() == 0: - text = item.text(1) - # Try to infer empty dict or list from context - if text == '': - # Try to guess from the key - key = item.text(0).strip().lower() - if key in ('globals'): - return {} # known dict-like keys - elif key.endswith('steps') or key.endswith('teardown_steps') or key.endswith('setup_steps'): - return [] # likely list-like - return text - # Check if it's a list (e.g. keys like [0], [1]) - is_list = all(item.child(j).text(0).startswith("[") for j in range(item.childCount())) - if is_list: - return [self.extract_item_data(item.child(j)) for j in range(item.childCount())] - else: - result = {} - for j in range(item.childCount()): - key = item.child(j).text(0) - value = self.extract_item_data(item.child(j)) - result[key] = value - return result - - def delete_selected_items(self): - selected_items = self.tree.selectedItems() - - for item in selected_items: - parent = item.parent() - if parent: - parent.removeChild(item) - else: - # Top-level item - index = self.indexOfTopLevelItem(item) - self.takeTopLevelItem(index) - # Manually trigger the handler after deletion - self.on_treeview_item_changed(item, 0) - - def sanitize_empty_fields(self, data: dict) -> dict: - # For RECIPE_HEADER_REQUIRED_FIELDS, force "globals" to be dict if empty string - if "globals" in data and (data["globals"] == "" or data["globals"] is None): - data["globals"] = {} - - # For sequences inside main_sequence or others, you can add similar logic: - seq_fields = ["setup_steps", "steps", "teardown_steps"] - for seq in seq_fields: - if seq in data and (data[seq] == "" or data[seq] is None): - data[seq] = [] - - # Similarly for dict fields in sequence: - dict_fields = ["parameters", "outputs", "locals", "input_mapping", "output_mapping"] - for dfield in dict_fields: - if dfield in data and (data[dfield] == "" or data[dfield] is None): - data[dfield] = {} - - return data - -# Misc - def log(self, message: str): - timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - self.log_console.append(f"[{timestamp}] {message}") - - def ask_save_invalid_file(self): - msg = QMessageBox(self) - msg.setIcon(QMessageBox.Warning) - msg.setWindowTitle("Save Corrupted Recipe") - msg.setText("The recipe content is invalid.\n" - "Do you want to save it anyway?\n " - "The contents of the Text View (right) would be saved.") - msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No) - msg.setDefaultButton(QMessageBox.No) - - # Force button text color to black - msg.setStyleSheet(""" - QPushButton { - color: black; - } - """) - - ret = msg.exec() + self.load_yaml_recipe(file_path) - if ret == QMessageBox.Yes: - return True - else: + def load_yaml_recipe(self, file_path) -> bool: + path = Path(file_path) + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + self.log(f"❌ Could not open {path}: {error}") return False + self.current_file_path = str(path) + self.reset_recovery_history() + self._set_text(text) + self.stacked_layout.setCurrentIndex(1) + parsed = self._validate_working_text(rebuild_sequencer=True) + self.setWindowTitle(f"Recipe Editor - {path.name}") + self.log(f"Loaded recipe text from: {path}") + return parsed.is_valid - def strip_star_prefix(self, data): - if isinstance(data, str): - if data.startswith("* "): - return data[2:] - return data - elif isinstance(data, list): - return [self.strip_star_prefix(item) for item in data] - elif isinstance(data, dict): - return {self.strip_star_prefix(k): self.strip_star_prefix(v) for k, v in data.items()} - else: - # Other types (int, float, bool, None, etc) returned as is - return data - - -# done 0.1 - VIEWER -# done 0.1 - Show some indicator that the changes are unsaved -# done 0.1 - allow to clear any field -# done 0.1 - indicate optional and required fields (with star?) -# done 0.1 - gray out save option if recipe is not opened -# done 0.1 - add button to automatically verify the recipe -# done 0.1 - right-hand side panel with descriptions on the selected field -# done 0.1 - Expert live view of the yaml file on the right hand side panel -# done 0.1 - on the log, show which line of the yaml was edited -# done 0.1 - Bugfix - make the right panel read only -# done 0.1 - Clear view shall be grayed if the view is already cleared (logo visible) -# done 0.1 - 1.0 unit tests - -# done 0.2 - GENERATION, EDITING, PARTIAL SUPPORT -# done 0.2 - allow saving -# done 0.2 - allow editing on the tree-view panel -# done 0.2 - allow editing on the yaml-editing panel -# done 0.2 - auto-verification on save, pop-up confirming save, when verification fails -# done 0.2 - add option -# done 0.2 - add CTRL+S shortcut -# done 0.2 - add recipe status indicator (only to show the current recipe state), inform if the treeview is not updated. -# done 0.2 - button toolbox to allow editing (just add few buttons and design the helper GUI) -# done 0.2 - add auto-conversion flag and disable it when the yaml have errors -# done 0.2 - add button allowing to revert to the last valid format (recreate last working state) -# done 0.2 - allow deletion of the whole steps -# done 0.2 - allow deletion of the whole sequences -# done 0.2 - print the faults found in recipe verification - on the status bar in the bottom -# done 0.2 - mark the text on red only if required + is not filled -# done 0.2 - allow full control over the YAML editor -# done 0.2 - bugfixing, unittests - -# done 1.0 - Create new recipe -# done 1.0 - fix the small gui imperfections -# done 1.0 - Some gui and UX improvements -# done 1.0 - Create new recipe from the template -# done 1.0 - derive YAML field descriptions from the production schema -# done 1.0 - Recipe interactive generator - one sequence, multiple steps -# done 1.0 - increase 1st column size -# done 1.0 - easy way to set the YamView application -# done 1.0 - change the required field to be a star or warning emoji, instead of red colour -# done 1.0 - check if it possible to fold only selected branches -# done 1.0 - bugfix - do not close previous recipe on cancelling the add_new -# done 1.0 - point automatically to the tree view on the yaml editor click -# done 1.0 - bugfix - fix the exceptions so they are parsed instead of shown -# done 1.0 - bugfix - exception on YAML edit failure -# done 1.0 - bugfix - if recipe is not saved, (just generated) we cannot close it -# done 1.0 - bugfix - if there is opened template recipe, trying to open and close, the title is wrong (try to reproduce first) -# done 1.0 - bugfix - title not updated on close recipe click -# done 1.0 - bugfix - invalid cleaning application state on close recipe click -# done 1.0 - bugfix - ask for saving invalid recipe in save as, the same way as save -# done 1.0 - bugfix - ensure, that the textview is saved -# done 1.0 - 1.0 unit tests - -# todo fix the module recognition, so we expect either file, path or name - + def on_close_recipe_clicked(self) -> None: + self.sequencer.clear() + self._set_text("") + self.stacked_layout.setCurrentIndex(0) + self.current_recipe = None + self.yaml_documents = [] + self.current_file_path = None + self.is_recipe_valid = False + self.reset_recovery_history() + self.setWindowTitle(f"{self.title} recipe editor") + self.show_recipe_info("No recipe open") + self._update_actions() -# todo 1.1 - if i delete whole recipe - its valid - well shout not be -# todo 1.1 - database of valid recipes -# todo 1.1 - include more information about what is missing in the structure -# todo 1.1 - bug [17/07/2025 12:00:41] ❌ Error in on_yaml_cursor_changed: Internal C++ object (PySide6.QtWidgets.QTreeWidgetItem) already deleted. -# todo 1.1 - generate the steps, but also have a way to recreate from template, based on instrument used or test type -# todo 1.1 - test with whitespaces, cross platform compatibility -# todo 1.1 - UX REFINEMENT -# todo 1.1 - Add possibility to open recent files -# todo 1.1 - Handle possibility that the recent file is not present anymore -# todo 1.1 - Add parsing of the recipe_version field and inform if the recipe is up to correct version -# todo 1.1 - Add recipe config file, where we can track the version (and maybe something more later) -# todo 1.1 - Bugfix - sometimes after opening new recipe for editing, the GUI is not refreshing (it does after clicking on the window) -# todo 1.1 - 1.1 unit tests -# todo 1.1 - autocomplete or helper to write down the tests + def reset_recovery_history(self) -> None: + self.last_valid_recipe = "" + self._update_actions() -# todo 1.2 - FULL SUPPORT -# todo 1.2 - creator - number of sequences, steps etc -# todo 1.2 - all fields programatically described in the yaml_description.py helper file -# todo 1.2 - clean up documentation -# todo 1.2 - add drop down lists on the treeview -# todo 1.2 - refined way of generating the recipe - the tree yaml view is not too intuitive + def keyPressEvent(self, event) -> None: + if event.key() == Qt.Key_Delete and self.sequencer.delete_selected(): + return + super().keyPressEvent(event) -# todo 2.0 - AI based yaml generation based on prompt (answers to fixed questions - how many sequences, how many steps etc) + def on_open_wiki_clicked(self) -> None: + webbrowser.open("https://acc-py.web.cern.ch/gitlab/pts/framework/pypts/docs/master/") -# todo 3.0 - check possibility to generate the recipe from the test plan + def on_open_gitlab_clicked(self) -> None: + webbrowser.open("https://gitlab.cern.ch/pts/framework/pypts") -# Nice features to show in 1.0: -# Shortcuts - redo, undo, -# YAML parser - if I put some undefined globals, it will define it for me -# Automatic recipe recovery -# Automatic cross-verification -# Verification on save, save as -# Unsaved changes -# Generate from the template + def log(self, message: str) -> None: + self.log_console.append(message) if __name__ == "__main__": - app = QApplication(sys.argv) window = RecipeEditorMainMenu() - window.show() - if len(sys.argv) > 1: - file_path = sys.argv[1] - window.current_file_path = file_path - print("Loaded a recipe from gui") - window.open_recipe() - + window.load_yaml_recipe(sys.argv[1]) sys.exit(app.exec()) diff --git a/src/pypts/YamVIEW/recipe_sequencer_setup.py b/src/pypts/YamVIEW/recipe_sequencer_setup.py index 085e682..bf36711 100644 --- a/src/pypts/YamVIEW/recipe_sequencer_setup.py +++ b/src/pypts/YamVIEW/recipe_sequencer_setup.py @@ -1,38 +1,54 @@ # SPDX-FileCopyrightText: 2025 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later +"""Sequence navigation and structured edit intents for YamVIEW.""" +from __future__ import annotations -from PySide6.QtWidgets import (QDialog, - QVBoxLayout, - QLabel, QHBoxLayout, QMessageBox,QWidget, - QAbstractItemView, QFrame, QToolBar, QApplication, QStyle, QListWidgetItem, QListWidget, - QSizePolicy) -from PySide6.QtCore import QSize, Qt, QPoint, Signal -from PySide6.QtGui import QAction,QDrag -from pypts.YamVIEW.recipe_step_setup import Step_setup, Skip_setup, Sequence_setup -from pypts.YamVIEW.styles import get_editor_theme_colors import re +from collections.abc import Callable +from typing import Any + +from PySide6.QtCore import QPoint, QSize, Qt, Signal +from PySide6.QtGui import QAction, QDrag +from PySide6.QtWidgets import ( + QAbstractItemView, + QApplication, + QDialog, + QFrame, + QHBoxLayout, + QLabel, + QListWidget, + QListWidgetItem, + QMessageBox, + QSizePolicy, + QStyle, + QToolBar, + QVBoxLayout, + QWidget, +) + +from pypts.YamVIEW.recipe_step_setup import Sequence_setup, Skip_setup, Step_setup +from pypts.YamVIEW.styles import get_editor_theme_colors + +FOLDER_TYPES = {"setup_folder", "main_folder", "teardown_folder"} + class StepBlock(QFrame): - def __init__(self, step_name, step_data, parent=None): + def __init__(self, step_name: str, step_data: dict[str, Any], parent=None): super().__init__(parent) self.step_name = step_name self.step_data = step_data self.setFrameShape(QFrame.StyledPanel) self.setObjectName("sequencerCard") self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - layout = QHBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) - layout.setSpacing(0) label = QLabel(step_name) label.setObjectName("sequencerStepTitle") - label.setWordWrap(False) label.setAlignment(Qt.AlignVCenter | Qt.AlignLeft) layout.addWidget(label) layout.addStretch() - self._label = label self.setMinimumHeight(max(34, label.sizeHint().height() + 10)) @@ -40,18 +56,13 @@ def _build_header_widget(text: str, indent: int) -> QFrame: container = QFrame() container.setObjectName("sequencerHeaderContainer") container.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - layout = QHBoxLayout(container) layout.setContentsMargins(indent + 8, 2, 8, 2) - layout.setSpacing(0) - label = QLabel(text) label.setObjectName("sequencerHeader") - label.setWordWrap(False) label.setAlignment(Qt.AlignVCenter | Qt.AlignLeft) layout.addWidget(label) layout.addStretch() - container.setMinimumHeight(max(32, label.sizeHint().height() + 12)) return container @@ -62,438 +73,381 @@ def _item_size_for(widget: QWidget) -> QSize: class SequencerWidget(QWidget): + """Display the recipe structure and emit model-independent edit intents.""" + def __init__(self, yaml_viewer=None, parent=None): super().__init__(parent) self.yaml_viewer = yaml_viewer - self.steps = [] - self.yaml_update_callback = None - self.expanded = False - self.new_sequence_request = None + self.steps: list[dict[str, Any]] = [] + self.yaml_update_callback: Callable[[list[dict[str, Any]]], None] | None = None self._dark = False + self.expanded = False + self._expanded_ids: set[str] = set() + self.current_setup_window: Step_setup | None = None - self.preamble_globals = {} - self.sequence_locals = {} - self.updated_preamble_globals = {} - self.updated_sequence_locals = {} - - # Main layout - self.layout = QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(0) - # ---- Toolbar ---- + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) self.Yamlbar = QToolBar() self.Yamlbar.setObjectName("yamSequencerToolbar") self.Yamlbar.setIconSize(QSize(22, 22)) - style = QApplication.style() - - # "Add Sequence" - act_new_seq = QAction(style.standardIcon(QStyle.SP_FileDialogNewFolder), - "Add Sequence Folder", self) - act_new_seq.triggered.connect(self.on_add_sequence) - self.Yamlbar.addAction(act_new_seq) - - # "Add Step" as a + icon - act_new_step = QAction("➕", self) - act_new_step.setToolTip("Make new step") - act_new_step.triggered.connect(self.on_add_step) - self.Yamlbar.addAction(act_new_step) - - act_disable_enable = QAction("±", self) - act_disable_enable.setToolTip("Disable/enable steps") - act_disable_enable.triggered.connect(self.on_change_state_step) - self.Yamlbar.addAction(act_disable_enable) - - # ---- List ---- - self.list_widget = StepListWidget() + self.action_add_sequence = QAction( + style.standardIcon(QStyle.SP_FileDialogNewFolder), "Add sequence", self + ) + self.action_add_step = QAction("➕", self) + self.action_add_step.setToolTip("Add step to the selected stage") + self.action_manage_steps = QAction("±", self) + self.action_manage_steps.setToolTip("Edit skip/error flags") + self.action_delete = QAction("Delete", self) + self.action_delete.setToolTip("Delete the selected step or sequence") + for action in ( + self.action_add_sequence, + self.action_add_step, + self.action_manage_steps, + self.action_delete, + ): + self.Yamlbar.addAction(action) + self.action_add_sequence.triggered.connect(self.on_add_sequence) + self.action_add_step.triggered.connect(self.on_add_step) + self.action_manage_steps.triggered.connect(self.on_change_state_step) + self.action_delete.triggered.connect(self.delete_selected) + + self.list_widget = StepListWidget(self) self.list_widget.setObjectName("sequencerList") - self.list_widget.setContentsMargins(0, 0, 0, 0) - self.list_widget.model().rowsMoved.connect(self.on_steps_reordered) self.list_widget.itemClicked.connect(self.on_item_clicked) self.list_widget.step_clicked.connect(self.navigate_to_step) self.list_widget.step_double_clicked.connect(self.edit_step) - - # Add widgets - self.layout.addWidget(self.Yamlbar) - self.layout.addWidget(self.list_widget) - - self.skip_warning = False + layout.addWidget(self.Yamlbar) + layout.addWidget(self.list_widget) self.set_dark(False) - def set_dark(self, dark: bool): + def set_dark(self, dark: bool) -> None: self._dark = dark colors = get_editor_theme_colors(dark) self.list_widget.setStyleSheet( "QListWidget#sequencerList {" f"background-color: {colors['surface_alt']};" f"border: 1px solid {colors['border']};" - "border-radius: 8px;" - "padding: 6px;" - "}" + "border-radius: 8px;padding: 6px;}" ) - def set_yaml_data(self, steps_list): + def set_yaml_data(self, steps_list: list[dict[str, Any]]) -> None: self.steps = steps_list self.refresh() - def refresh(self): - """Render a fully nested collapsible tree of steps.""" + def _node_key(self, node: dict[str, Any]) -> str: + return str(node.get("_id") or node.get("_sequence_id") or node.get("steptype")) + + def refresh(self) -> None: + """Render sequence/stage headers and leaf steps from the working structure.""" + selected = self.current_node() + selected_key = self._node_key(selected) if selected else None self.list_widget.clear() - def add_step_item(step, indent=0): - step_type = step.get("steptype", "") - step_name = step.get("step_name", "Unnamed Step") - children = step.get("children", []) - - - # If the step has children → treat as collapsible folder - if children: - - # Folder header - prefix = "➖" if self.expanded else "➕" - header_item = QListWidgetItem(f"{prefix} {step_name}") - header_item.setFlags(Qt.ItemIsEnabled) - header_item.setData(Qt.UserRole, step) - self.list_widget.addItem(header_item) - - # Indent header - header_item.setData(Qt.UserRole + 2, indent) - header_widget = _build_header_widget(f"{prefix} {step_name}", indent) - header_item.setSizeHint(_item_size_for(header_widget)) - self.list_widget.setItemWidget(header_item, header_widget) - - # Add all children recursively - child_items = [] - for child in children: - child_index_start = self.list_widget.count() - add_step_item(child, indent + 20) - for i in range(child_index_start, self.list_widget.count()): - child_items.append(self.list_widget.item(i)) - - # Save child references for collapse/expand - header_item.setData(Qt.UserRole + 1, child_items) - - # Hide children initially if folder not expanded - if not self.expanded: - for c in child_items: - c.setHidden(True) - widget = self.list_widget.itemWidget(c) - if widget: - widget.setVisible(False) + def add_node(node: dict[str, Any], indent: int = 0) -> None: + is_folder = node.get("steptype") in FOLDER_TYPES | {"sequence_folder"} + if is_folder: + key = self._node_key(node) + expanded = self.expanded or key in self._expanded_ids + prefix = "➖" if expanded else "➕" + item = QListWidgetItem() + item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable) + item.setData(Qt.UserRole, node) + item.setData(Qt.UserRole + 2, indent) + widget = _build_header_widget(f"{prefix} {node['step_name']}", indent) + item.setSizeHint(_item_size_for(widget)) + self.list_widget.addItem(item) + self.list_widget.setItemWidget(item, widget) + descendants: list[QListWidgetItem] = [] + for child in node.get("children", []): + start = self.list_widget.count() + add_node(child, indent + 20) + descendants.extend( + self.list_widget.item(index) + for index in range(start, self.list_widget.count()) + ) + item.setData(Qt.UserRole + 1, descendants) + if not expanded: + for descendant in descendants: + descendant.setHidden(True) + return + if node.get("steptype") == "preamble": + item = QListWidgetItem() + item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable) + item.setData(Qt.UserRole, node) + widget = _build_header_widget(node["step_name"], indent) else: - # Leaf step → StepBlock - block = StepBlock(step_name, step) - container = QFrame() - container_layout = QHBoxLayout(container) - container.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - container_layout.setContentsMargins(indent + 4, 2, 4, 2) - container_layout.setSpacing(0) - container_layout.setAlignment(Qt.AlignLeft) - container_layout.addWidget(block) - container.setMinimumHeight(block.minimumHeight() + 4) item = QListWidgetItem() - item.setSizeHint(_item_size_for(container)) - self.list_widget.addItem(item) - self.list_widget.setItemWidget(item, container) - item.setData(Qt.UserRole, step) - - # Build the full tree - for step in self.steps: - add_step_item(step, indent=0) - - - def on_steps_reordered(self, parent, start, end, destination, row): - """Update folder order internally; don't touch YAML yet.""" - # Iterate visible items and reorder them inside their folder - sequence_id = None - for i in range(self.list_widget.count()): - item = self.list_widget.item(i) - step_data = item.data(Qt.UserRole) - if step_data and "_sequence_id" in step_data: - sequence_id = step_data["_sequence_id"] - break - - if sequence_id is None: - print("No sequence_id found during reorder.") - return + item.setFlags( + Qt.ItemIsEnabled + | Qt.ItemIsSelectable + | Qt.ItemIsDragEnabled + | Qt.ItemIsDropEnabled + ) + item.setData(Qt.UserRole, node) + block = StepBlock(node.get("step_name", "Unnamed step"), node) + widget = QFrame() + row = QHBoxLayout(widget) + row.setContentsMargins(indent + 4, 2, 4, 2) + row.addWidget(block) + widget.setMinimumHeight(block.minimumHeight() + 4) + item.setSizeHint(_item_size_for(widget)) + self.list_widget.addItem(item) + self.list_widget.setItemWidget(item, widget) + if selected_key == self._node_key(node): + self.list_widget.setCurrentItem(item) + + for node in self.steps: + add_node(node) + + def current_node(self) -> dict[str, Any] | None: + item = self.list_widget.currentItem() + return item.data(Qt.UserRole) if item else None + + def _sequence(self, sequence_id: str) -> dict[str, Any] | None: + return next( + (node for node in self.steps if node.get("_sequence_id") == sequence_id), + None, + ) - folder_map = {} - for seq in self.steps: - if seq.get("_sequence_id") == sequence_id: - for folder in seq.get("children", []): - folder_map[folder["steptype"]] = folder - break - - for i in range(self.list_widget.count()): - item = self.list_widget.item(i) - step_data = item.data(Qt.UserRole) - if not step_data: - continue - - parent_type = step_data.get("_parent") - if parent_type: - folder = folder_map.get(parent_type) - if folder: - children = folder.get("children", []) - # Remove & append to maintain new order - if step_data in children: - children.remove(step_data) - children.append(step_data) + def _folder(self, sequence_id: str, folder_type: str) -> dict[str, Any] | None: + sequence = self._sequence(sequence_id) + if sequence is None: + return None + return next( + (child for child in sequence["children"] if child.get("steptype") == folder_type), + None, + ) + + def _selected_destination(self) -> tuple[str, str] | None: + node = self.current_node() + if node and node.get("steptype") in FOLDER_TYPES: + return node["_sequence_id"], node["steptype"] + if node and node.get("_parent"): + return node["_sequence_id"], node["_parent"] + if node and node.get("steptype") == "sequence_folder": + return node["_sequence_id"], "main_folder" + sequence = next( + (item for item in self.steps if item.get("steptype") == "sequence_folder"), + None, + ) + if sequence: + return sequence["_sequence_id"], "main_folder" + return None + def _notify(self) -> bool: + self.refresh() if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) - - def move_step_to_folder(self, step, old_parent, new_parent, old_seq_id, new_seq_id): - print(f"Moving: {step['step_name']} {old_parent} → {new_parent}") - # Find actual folder dicts - source_folder = None - dest_folder = None - - for seq in self.steps: - if seq.get("_sequence_id") != old_seq_id: - continue - - for folder in seq["children"]: - if folder["steptype"] == old_parent: - source_folder = folder - break - - for seq in self.steps: - if seq.get("_sequence_id") != new_seq_id: - continue - for folder in seq["children"]: - if folder["steptype"] == new_parent: - dest_folder = folder - break - if not source_folder or not dest_folder: - print("Folder lookup failed (source or destination missing).") - return + return self.yaml_update_callback(self.steps) is not False + return True - try: - source_folder["children"].remove(step) - except ValueError: - print("Step not found in source folder. Possibly already moved or inconsistent tree.") + def on_item_clicked(self, item: QListWidgetItem) -> None: + node = item.data(Qt.UserRole) + if node.get("steptype") not in FOLDER_TYPES | {"sequence_folder"}: return - - - step["_parent"] = new_parent - step["_sequence_id"] = new_seq_id - dest_folder["children"].append(step) + key = self._node_key(node) + if key in self._expanded_ids: + self._expanded_ids.remove(key) + else: + self._expanded_ids.add(key) self.refresh() - if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) - - def on_item_clicked(self, item): - """Toggle collapsible folder visibility.""" - children = item.data(Qt.UserRole + 1) - if not children: - return # Not a folder - - - hidden = children[0].isHidden() # toggle state - for child_item in children: - child_item.setHidden(not hidden) - - # Update header icon - if hidden: - item.setText(f"➖ {item.data(Qt.UserRole).get('step_name', 'Unnamed Step')}") - self.expanded = True - else: - item.setText(f"➕ {item.data(Qt.UserRole).get('step_name', 'Unnamed Step')}") - self.expanded = False - - def navigate_to_step(self, step_data): - if not self.yaml_viewer: + def navigate_to_step(self, step_data: dict[str, Any]) -> None: + if not self.yaml_viewer or "_node" not in step_data: return - step_name = step_data.get("step_name", "") - - yaml_text = self.yaml_viewer.toPlainText() - cursor = self.yaml_viewer.textCursor() - - # Try line number if stored - id_number = step_data.get("_id") - if id_number is not None: - step_type = step_data.get("steptype", "") - step_name = step_data.get("step_name", "") - step_description = step_data.get("description", "") - - block_pattern = ( - r"steptype:\s*" + re.escape(step_type) + r"\s*" - r"step_name:\s*" + re.escape(step_name) + r"\s*" - r"description:\s*" + re.escape(step_description) + node = step_data["_node"] + pattern = ( + r"steptype:\s*" + re.escape(node.get("steptype", "")) + + r"[\s\S]*?step_name:\s*" + re.escape(node.get("step_name", "")) ) - match = re.search(block_pattern, yaml_text) - if match: - position = match.start() - cursor.setPosition(position) - else: - # fallback: search by step_name - step_name = step_data.get("step_name", "") - position = yaml_text.find(step_name) - if position == -1: - return - cursor.setPosition(position) - - self.yaml_viewer.setTextCursor(cursor) - self.yaml_viewer.setFocus() - - def edit_step(self, step_data): - - if hasattr(self, "current_setup_window") and self.current_setup_window: + match = re.search(pattern, self.yaml_viewer.toPlainText()) + if match: + cursor = self.yaml_viewer.textCursor() + cursor.setPosition(match.start()) + self.yaml_viewer.setTextCursor(cursor) + self.yaml_viewer.setFocus() + + def on_add_sequence(self) -> None: + dialog = Sequence_setup(parent=self) + if not dialog.exec(): + return + data = dialog.result_sequence + sequence_id = data["sequence_name"] + suffix = 2 + while self._sequence(sequence_id): + sequence_id = f"{data['sequence_name']}#{suffix}" + suffix += 1 + self.steps.append(_sequence_node(data, sequence_id)) + self._expanded_ids.add(sequence_id) + self._notify() + + def on_add_step(self) -> None: + destination = self._selected_destination() + if destination is None: + QMessageBox.warning(self, "No sequence", "Add a sequence before adding steps.") + return + dialog = Step_setup(parent=self) + dialog._skip_warning = True + if not dialog.exec(): + return + sequence_id, parent_type = destination + folder = self._folder(sequence_id, parent_type) + if folder is None: + return + step = dialog.result_step + step["_parent"] = parent_type + step["_sequence_id"] = sequence_id + folder["children"].append(step) + self._expanded_ids.update({sequence_id, f"{sequence_id}:{parent_type}"}) + self._notify() + + def edit_step(self, step_data: dict[str, Any]) -> None: + if "_node" not in step_data or not step_data.get("_parent"): + return + if self.current_setup_window: self.current_setup_window.close() self.current_setup_window.deleteLater() - self.current_setup_window = None - # Existing canonical data is rendered by the same schema-driven form - # used for new steps. - node = step_data.get("_node", {}) - step_id = step_data.get("_id", None) - - self.current_setup_window = Step_setup() - self.current_setup_window.AlreadyID = step_id + dialog = Step_setup(parent=self) + dialog.AlreadyID = step_data["_id"] try: - self.current_setup_window.load_definition(node) + dialog.load_definition(step_data["_node"]) except (KeyError, ValueError) as error: - QMessageBox.warning(self, "Invalid Step", str(error)) + QMessageBox.warning(self, "Invalid step", str(error)) return - self.current_setup_window.finished.connect( - lambda result: self.on_edit_window_closed(result) + self.current_setup_window = dialog + dialog.finished.connect( + lambda result, original=step_data: self._finish_edit(result, original) ) - self.current_setup_window.show() - - def on_edit_window_closed(self, result): - if result == QDialog.Accepted: - #self.skip_warning = self.current_setup_window.skip_checkbox - self.updatingYAMLFormat(self.current_setup_window, edit_child=True) - print("Ran after closing (OK pressed)") - self.refresh() - - def loaded_step_parameters(self, step_name, node, method = None, gui_name= None): - - if gui_name: - self.current_setup_window._skip_warning = True - self.current_setup_window.list_steptype.setCurrentText(gui_name) - self.current_setup_window._skip_warning = False - self.current_setup_window.list_steptype.setCurrentText(step_name) - self.current_setup_window.step_name_input.setText(step_name) - self.current_setup_window.description_input.setText(node.get("description", "")) - self.current_setup_window.skip_checkbox.setChecked(node.get("skip", False)) - self.current_setup_window.continue_on_error_checkbox.setChecked(node.get("continue_on_error", False)) - self.current_setup_window.setWindowTitle(f"Edit Step: {step_name}") - if method: - self.current_setup_window.list_actiontypes.setCurrentText(node.get("action_type", "method")) - self.current_setup_window.module_input.setText(node.get("module", "")) - self.current_setup_window.method_input.setText(node.get("method_name", "")) - - def receive_globals(self, globals_dict: dict): - """Update the internal globals reference.""" - self.preamble_globals = globals_dict - - def update_global_value(self, key, value): - """Update or add a global value.""" - self.updated_preamble_globals[key] = value - - def receive_locals(self, globals_dict: dict): - """Update the internal locals reference.""" - self.sequence_locals = globals_dict - - def update_local_value(self, key, value): - """Update or add a local value.""" - self.updated_sequence_locals[key] = value - - def on_add_sequence(self): - - _sequence = Sequence_setup(self) - if _sequence.exec(): - self.updatingYAMLFormat(_sequence) - - def on_add_step(self): - _step = Step_setup(self) - _step._skip_warning = True - if _step.exec(): # blocks until OK or Cancel - self.updatingYAMLFormat(_step) - - def on_change_state_step(self): - - steps = self.steps[1]["children"][1]["children"] - - dialog = Skip_setup(steps=self.steps) - if dialog.exec_(): - self.updatingYAMLFormat(dialog, edit_child=True) - - def updatingYAMLFormat(self, _step, edit_child = False): - new_step = getattr(_step, "result_step", None) - new_sequence = getattr(_step, "result_sequence", None) - - if hasattr(_step, "global_variables") and _step.global_variables: - for key, value in _step.global_variables.items(): - self.update_global_value(key, value) - - if hasattr(_step, "local_variables") and _step.local_variables: - for key, value in _step.local_variables.items(): - self.update_local_value(key, value) - - if new_step and not edit_child: - # Assign parent folder type before inserting - new_step["_parent"] = "main_folder" - - self.steps[1]["children"][1]["children"].append(new_step) - - self.refresh() - - if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) - - elif new_step and edit_child: - - new_step["_parent"] = "main_folder" - new_ID = new_step.get("_id") - - for idx, child in enumerate(self.steps[1]["children"][1]["children"]): - if child.get("_id").strip() == str(new_ID).strip(): - self.steps[1]["children"][1]["children"][idx] = new_step - break - else: - self.steps[1]["children"][1]["children"].append(new_step) + dialog.show() - self.refresh() - - if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) - if new_sequence: - - self.steps.append(new_sequence) - self.new_sequence_request = new_sequence - - self.refresh() - - if callable(self.yaml_update_callback): - self.yaml_update_callback(self.steps) + def _finish_edit(self, result: int, original: dict[str, Any]) -> None: + dialog = self.current_setup_window + if result != QDialog.Accepted or dialog is None: + return + replacement = dialog.result_step + replacement["_parent"] = original["_parent"] + replacement["_sequence_id"] = original["_sequence_id"] + folder = self._folder(original["_sequence_id"], original["_parent"]) + if folder is None: + return + for index, child in enumerate(folder["children"]): + if child.get("_id") == original.get("_id"): + folder["children"][index] = replacement + self._notify() + return - def clear(self): - """Completely removes all widgets and layouts from self.container_layout.""" + def move_step( + self, + step: dict[str, Any], + sequence_id: str, + parent_type: str, + index: int | None = None, + ) -> bool: + source = self._folder(step.get("_sequence_id", ""), step.get("_parent", "")) + destination = self._folder(sequence_id, parent_type) + if source is None or destination is None or step not in source["children"]: + return False + source["children"].remove(step) + step["_sequence_id"] = sequence_id + step["_parent"] = parent_type + if index is None: + destination["children"].append(step) + else: + destination["children"].insert(index, step) + return self._notify() - lw = self.list_widget + def move_step_to_folder( + self, step, old_parent, new_parent, old_seq_id, new_seq_id + ) -> None: + self.move_step(step, new_seq_id, new_parent) - lw.blockSignals(True) - # Remove widgets first - for i in range(lw.count()): - item = lw.item(i) - widget = lw.itemWidget(item) - if widget: - widget.setParent(None) - widget.deleteLater() + def on_steps_reordered(self, *args) -> None: + """Compatibility entry point; drag/drop commits through :meth:`move_step`.""" - # Now clear the QListWidgetItem objects - lw.clear() + def on_change_state_step(self) -> None: + if not any(item.get("steptype") == "sequence_folder" for item in self.steps): + return + dialog = Skip_setup(self.steps, self) + if dialog.exec(): + self._notify() + + def delete_selected(self, confirm: bool = True) -> bool: + node = self.current_node() + return self.delete_node(node, confirm=confirm) + + def delete_node(self, node: dict[str, Any] | None, confirm: bool = True) -> bool: + """Delete one identified step or sequence; virtual folders are protected.""" + if node is None: + return False + node_type = node.get("steptype") + if node_type == "preamble" or node_type in FOLDER_TYPES: + return False + if confirm: + label = node.get("step_name", "selected item") + answer = QMessageBox.question( + self, + "Delete item", + f"Delete '{label}'?", + QMessageBox.Yes | QMessageBox.No, + QMessageBox.No, + ) + if answer != QMessageBox.Yes: + return False + if node_type == "sequence_folder": + self.steps.remove(node) + else: + folder = self._folder(node.get("_sequence_id", ""), node.get("_parent", "")) + if folder is None or node not in folder["children"]: + return False + folder["children"].remove(node) + return self._notify() - # Clear internal drag/selection state - lw.clearSelection() - lw._mouse_press_pos = None - lw.blockSignals(False) + def clear(self) -> None: + self.steps = [] + self._expanded_ids.clear() + self.list_widget.clear() +def _sequence_node(data: dict[str, Any], sequence_id: str) -> dict[str, Any]: + """Create YamVIEW navigation metadata around one plain sequence document.""" + folders = [] + for title, folder_type, field_name in ( + ("Setup Steps", "setup_folder", "setup_steps"), + ("Main Steps", "main_folder", "steps"), + ("Teardown Steps", "teardown_folder", "teardown_steps"), + ): + children = [] + for index, node in enumerate(data.get(field_name, [])): + stable_id = node.get("id") or f"{sequence_id}:{folder_type}:{index}" + children.append( + { + "step_name": node["step_name"], + "steptype": node["steptype"], + "_node": node, + "_parent": folder_type, + "_sequence_id": sequence_id, + "_id": stable_id, + } + ) + folders.append( + { + "step_name": title, + "steptype": folder_type, + "children": children, + "_sequence_id": sequence_id, + "_id": f"{sequence_id}:{folder_type}", + } + ) + return { + "step_name": f"Sequence: {data['sequence_name']}", + "steptype": "sequence_folder", + "children": folders, + "_node": data, + "_sequence_id": sequence_id, + "_id": sequence_id, + } class StepListWidget(QListWidget): @@ -506,100 +460,68 @@ def __init__(self, parent=None): self.setDefaultDropAction(Qt.MoveAction) self.setSelectionMode(QListWidget.SingleSelection) self.setAcceptDrops(True) - self.setDropIndicatorShown(True) + self.setDropIndicatorShown(True) self.setDragEnabled(True) - self._mouse_press_pos = None + self._mouse_press_pos = QPoint() - def startDrag(self, supportedActions): + @property + def sequencer(self) -> SequencerWidget: + return self.parent() + + def startDrag(self, supported_actions) -> None: item = self.currentItem() - if not item: + node = item.data(Qt.UserRole) if item else None + if not node or not node.get("_parent"): return - widget = self.itemWidget(item) - if not widget: - return - - # Create a pixmap of the widget (the “ghost” that follows the mouse) - label_widget = widget.findChild(QLabel) - if label_widget: - pixmap_widget = label_widget.parentWidget() - else: - pixmap_widget = widget - pixmap = pixmap_widget.grab() - #pixmap = widget.grab() - # Optionally make it semi-transparent - pixmap.setDevicePixelRatio(widget.devicePixelRatioF()) - - # Create the drag object drag = QDrag(self) - selected_items = self.selectedItems() - mime_data = self.mimeData(selected_items) # <-- important! - drag.setMimeData(mime_data) - drag.setPixmap(pixmap) - drag.setHotSpot(QPoint(pixmap.width() // 2, pixmap.height() // 2)) - - # Start the drag + drag.setMimeData(self.mimeData([item])) + if widget: + pixmap = widget.grab() + drag.setPixmap(pixmap) + drag.setHotSpot(QPoint(pixmap.width() // 2, pixmap.height() // 2)) drag.exec(Qt.MoveAction) - - def mousePressEvent(self, event): + + def mousePressEvent(self, event) -> None: self._mouse_press_pos = event.position().toPoint() super().mousePressEvent(event) - def mouseReleaseEvent(self, event): - delta = (event.position().toPoint() - self._mouse_press_pos).manhattanLength() - if delta < QApplication.startDragDistance(): + def mouseReleaseEvent(self, event) -> None: + if (event.position().toPoint() - self._mouse_press_pos).manhattanLength() < QApplication.startDragDistance(): item = self.itemAt(event.position().toPoint()) - if item: - step_data = item.data(Qt.UserRole) - if step_data: - self.step_clicked.emit(step_data) # notify parent + node = item.data(Qt.UserRole) if item else None + if node: + self.step_clicked.emit(node) super().mouseReleaseEvent(event) - - def mouseDoubleClickEvent(self, event): + + def mouseDoubleClickEvent(self, event) -> None: item = self.itemAt(event.position().toPoint()) - if item: - step_data = item.data(Qt.UserRole) - if step_data: - self.step_double_clicked.emit(step_data) + node = item.data(Qt.UserRole) if item else None + if node: + self.step_double_clicked.emit(node) super().mouseDoubleClickEvent(event) - def dropEvent(self, event): - pos = event.position().toPoint() if hasattr(event, "position") else event.pos() - target_item = self.itemAt(pos) - - if not target_item: - return super().dropEvent(event) - + def dropEvent(self, event) -> None: dragged_item = self.currentItem() - if not dragged_item: - return super().dropEvent(event) - - dragged_step = dragged_item.data(Qt.UserRole) - target_step = target_item.data(Qt.UserRole) - - - # Only handle drop ON a folder header - if target_step and "children" in target_step: - new_parent = target_step["steptype"] - old_parent = dragged_step["_parent"] - - new_seq_id = target_step.get("_sequence_id") - old_seq_id = dragged_step.get("_sequence_id") - - if new_parent != old_parent or new_seq_id != old_seq_id: - sequencer = self.parent() - if hasattr(sequencer, "move_step_to_folder"): - sequencer.move_step_to_folder( - step=dragged_step, - old_parent=old_parent, - new_parent=new_parent, - old_seq_id=old_seq_id, - new_seq_id=new_seq_id, - ) - - event.accept() + target_item = self.itemAt(event.position().toPoint()) + dragged = dragged_item.data(Qt.UserRole) if dragged_item else None + target = target_item.data(Qt.UserRole) if target_item else None + if not dragged or not dragged.get("_parent") or not target: + event.ignore() return - - # Otherwise fallback to normal same-folder reorder - return super().dropEvent(event) - + if target.get("steptype") in FOLDER_TYPES: + sequence_id = target["_sequence_id"] + parent_type = target["steptype"] + index = None + elif target.get("_parent"): + sequence_id = target["_sequence_id"] + parent_type = target["_parent"] + folder = self.sequencer._folder(sequence_id, parent_type) + index = folder["children"].index(target) if folder else None + else: + event.ignore() + return + if self.sequencer.move_step(dragged, sequence_id, parent_type, index): + event.acceptProposedAction() + else: + event.ignore() diff --git a/src/pypts/YamVIEW/recipe_step_setup.py b/src/pypts/YamVIEW/recipe_step_setup.py index f7a30fa..4437861 100644 --- a/src/pypts/YamVIEW/recipe_step_setup.py +++ b/src/pypts/YamVIEW/recipe_step_setup.py @@ -1,21 +1,33 @@ # SPDX-FileCopyrightText: 2025 CERN # # SPDX-License-Identifier: LGPL-2.1-or-later +"""Schema-driven dialogs used by the YamVIEW recipe editor.""" + +from __future__ import annotations -from PySide6.QtWidgets import (QDialog,QFileDialog, QComboBox, - QVBoxLayout, - QLabel, - QDialogButtonBox, - QPushButton,QLineEdit,QTextEdit,QCheckBox, QHBoxLayout, QMessageBox,QWidget,QTableWidgetItem,QTableWidget, - QAbstractItemView, QScrollArea) -import os, uuid import json +import uuid from typing import Any -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError +from PySide6.QtWidgets import ( + QCheckBox, + QComboBox, + QDialog, + QDialogButtonBox, + QHBoxLayout, + QLabel, + QLineEdit, + QMessageBox, + QPushButton, + QScrollArea, + QTextEdit, + QVBoxLayout, + QWidget, +) from pypts.recipe_language import Recipe as RecipeDefinition -from pypts.recipe_language import StepDefinition +from pypts.recipe_language import Sequence, StepDefinition def recipe_form_schema() -> dict[str, Any]: @@ -34,20 +46,27 @@ def resolve_schema(schema: dict[str, Any], node: dict[str, Any]) -> dict[str, An return node -def discriminator_schemas(name: str, schema: dict[str, Any] | None = None): +def discriminator_schemas( + name: str, schema: dict[str, Any] | None = None +) -> dict[str, dict[str, Any]]: """Map discriminator values to their resolved model schemas.""" schema = schema or recipe_form_schema() definition = schema["$defs"][name] mapping = definition["discriminator"]["mapping"] - return {key: resolve_schema(schema, {"$ref": reference}) for key, reference in mapping.items()} + return { + key: resolve_schema(schema, {"$ref": reference}) + for key, reference in mapping.items() + } def schema_widget_kind(field: dict[str, Any]) -> str: - """Select a primitive or structured editor solely from JSON Schema type.""" + """Select a control solely from JSON Schema metadata and JSON value type.""" if "enum" in field or "const" in field: return "choice" variants = field.get("anyOf", []) kinds = {item.get("type") for item in variants} + if "$ref" in field or any("$ref" in item for item in variants): + return "structured" kind = field.get("type") if kind == "boolean": return "boolean" @@ -60,11 +79,13 @@ def schema_widget_kind(field: dict[str, Any]) -> str: return "text" -def recipe_form_description(schema: dict[str, Any] | None = None) -> dict[str, Any]: +def recipe_form_description( + schema: dict[str, Any] | None = None, +) -> dict[str, dict[str, Any]]: """Describe selectors and fields directly from the production JSON Schema.""" schema = schema or recipe_form_schema() - def variants(name: str): + def variants(name: str) -> dict[str, Any]: result = {} for discriminator, definition in discriminator_schemas(name, schema).items(): required = set(definition.get("required", [])) @@ -96,6 +117,7 @@ def build_schema_widget(field: dict[str, Any], parent=None): return DiscriminatedMappingWidget("inputs", parent) if mapping_reference.endswith("/OutputMapping"): return DiscriminatedMappingWidget("outputs", parent) + kind = schema_widget_kind(field) if kind == "choice": widget = QComboBox(parent) @@ -114,6 +136,7 @@ def build_schema_widget(field: dict[str, Any], parent=None): widget = QLineEdit(parent) if "default" in field and field["default"] is not None: widget.setText(str(field["default"])) + details = field.get("description", "") if field.get("examples"): details += f" Example: {field['examples'][0]!r}." @@ -122,12 +145,12 @@ def build_schema_widget(field: dict[str, Any], parent=None): class SchemaFormWidget(QWidget): - """Generic YamVIEW form whose fields come entirely from JSON Schema.""" + """Generic form whose fields and defaults come entirely from JSON Schema.""" def __init__(self, variant: dict[str, Any], parent=None): super().__init__(parent) self.variant = variant - self.field_widgets = {} + self.field_widgets: dict[str, QWidget] = {} layout = QVBoxLayout(self) description = variant.get("description") if description: @@ -142,8 +165,8 @@ def __init__(self, variant: dict[str, Any], parent=None): layout.addWidget(widget) def values(self) -> dict[str, Any]: - """Read recipe step-definition values from the schema controls.""" - values = {} + """Read Python values from the schema controls.""" + values: dict[str, Any] = {} fields = self.variant["fields"] for name, widget in self.field_widgets.items(): field = fields[name] @@ -160,19 +183,29 @@ def values(self) -> dict[str, Any]: continue value = field.get("default") else: - value = json.loads(text) + try: + value = json.loads(text) + except json.JSONDecodeError as error: + widget.setFocus() + raise ValueError( + f"Field '{name}' must contain valid JSON: {error.msg}." + ) from error else: text = widget.text().strip() if not text and not field.get("required"): continue value = text if field.get("type") in {"integer", "number"}: - value = json.loads(text) + try: + value = json.loads(text) + except json.JSONDecodeError as error: + widget.setFocus() + raise ValueError(f"Field '{name}' must be a number.") from error values[name] = value return values def load_values(self, values: dict[str, Any]) -> None: - """Populate controls from an existing canonical step mapping.""" + """Populate controls from canonical recipe values.""" for name, value in values.items(): widget = self.field_widgets.get(name) if widget is None: @@ -192,19 +225,19 @@ def load_values(self, values: dict[str, Any]) -> None: class DiscriminatedMappingWidget(QWidget): - """Editable mapping rows driven by an input/output discriminator schema.""" + """Editable input/output rows driven by discriminator schema mappings.""" def __init__(self, mapping_kind: str, parent=None): super().__init__(parent) self.variants = recipe_form_description()[mapping_kind] - self.rows = [] + self.rows: list[dict[str, Any]] = [] self.layout = QVBoxLayout(self) add_button = QPushButton("Add mapping") add_button.clicked.connect(lambda: self.add_row()) self.layout.addWidget(add_button) @staticmethod - def _clear(layout): + def _clear(layout) -> None: while layout.count(): item = layout.takeAt(0) if item.widget() is not None: @@ -212,8 +245,8 @@ def _clear(layout): elif item.layout() is not None: DiscriminatedMappingWidget._clear(item.layout()) - def add_row(self, name="", value=None): - value = value or {} + def add_row(self, name: str = "", value: dict[str, Any] | None = None) -> None: + original_value = value or {} row_widget = QWidget(self) row_layout = QVBoxLayout(row_widget) heading = QHBoxLayout() @@ -222,9 +255,8 @@ def add_row(self, name="", value=None): name_edit.setText(name) type_combo = QComboBox(row_widget) type_combo.addItems(self.variants) - mapping_type = value.get("type") - if mapping_type in self.variants: - type_combo.setCurrentText(mapping_type) + if original_value.get("type") in self.variants: + type_combo.setCurrentText(original_value["type"]) remove_button = QPushButton("Remove", row_widget) heading.addWidget(name_edit) heading.addWidget(type_combo) @@ -238,12 +270,12 @@ def add_row(self, name="", value=None): "type": type_combo, "form_layout": form_layout, "form": None, - "value": value, + "value": original_value, } self.rows.append(row) self.layout.insertWidget(self.layout.count() - 1, row_widget) - def render(discriminator): + def render(discriminator: str) -> None: self._clear(form_layout) form = SchemaFormWidget(self.variants[discriminator], row_widget) row["form"] = form @@ -252,7 +284,7 @@ def render(discriminator): form.load_values(row["value"]) row["value"] = {} - def remove(): + def remove() -> None: self.rows.remove(row) row_widget.setParent(None) row_widget.deleteLater() @@ -261,18 +293,22 @@ def remove(): remove_button.clicked.connect(remove) render(type_combo.currentText()) - def values(self): - result = {} + def values(self) -> dict[str, Any]: + result: dict[str, Any] = {} for row in self.rows: name = row["name"].text().strip() if not name: + row["name"].setFocus() raise ValueError("Mapping rows require a name.") + if name in result: + row["name"].setFocus() + raise ValueError(f"Mapping name '{name}' is duplicated.") value = row["form"].values() value["type"] = row["type"].currentText() result[name] = value return result - def load_values(self, values): + def load_values(self, values: dict[str, Any]) -> None: for row in tuple(self.rows): self.rows.remove(row) row["widget"].setParent(None) @@ -280,684 +316,246 @@ def load_values(self, values): for name, value in values.items(): self.add_row(name, value) + class Sequence_setup(QDialog): - def __init__(self, steps, parent=None): + """Small product-oriented dialog for creating a validated empty sequence.""" + + def __init__(self, steps=None, parent=None): super().__init__(parent) self.setWindowTitle("New Sequence Setup") self.resize(600, 400) - - main_layout = QVBoxLayout(self) - + layout = QVBoxLayout(self) self.sequence_name_input = QLineEdit() - self.sequence_name_input.setPlaceholderText("New Sequence name") - main_layout.addWidget(QLabel("Sequence name")) - main_layout.addWidget(self.sequence_name_input) - + self.sequence_name_input.setPlaceholderText("New sequence name") + layout.addWidget(QLabel("Sequence name *")) + layout.addWidget(self.sequence_name_input) self.description_input = QTextEdit() self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholdertext("describe the test here") - main_layout.addWidget(QLabel("Description")) - main_layout.addWidget(self.description_input) - - - main_layout.addStretch() - + self.description_input.setPlaceholderText("Describe the sequence") + layout.addWidget(QLabel("Description *")) + layout.addWidget(self.description_input) + layout.addStretch() buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - main_layout.addWidget(buttons) buttons.accepted.connect(self.accept) buttons.rejected.connect(self.reject) - - - def accept(self): - + layout.addWidget(buttons) - validate_method = f"validate_sequence" - if hasattr(self, validate_method): - ok, error_msg = getattr(self, validate_method)() - if not ok: - self.setStyleSheet("""QMessageBox QPushButton { color: black;}""") - msg = QMessageBox.warning(self,"Missing data", error_msg) - return - - self.result_sequence = { - "sequence_name": self.sequence_name_input.text(), - "description": self.description_input.toPlainText(), + def accept(self) -> None: + data = { + "sequence_name": self.sequence_name_input.text().strip(), + "description": self.description_input.toPlainText().strip(), "parameters": {}, - "locals": {}, "outputs": {}, - "setup_steps": {}, - "steps": {}, - "teardown_steps": {}, + "locals": {}, + "setup_steps": [], + "steps": [], + "teardown_steps": [], } - + try: + definition = Sequence.model_validate(data) + except ValidationError as error: + QMessageBox.warning(self, "Invalid sequence", str(error)) + return + self.result_sequence = definition.model_dump( + mode="python", by_alias=True, exclude_none=True + ) super().accept() - def validate_sequence(self): - if not self.sequence_name_input.text().strip(): - return False, "Step must have a name" - - return True, "" - class Skip_setup(QDialog): + """Bulk editor for schema-defined skip and continue-on-error fields.""" + def __init__(self, steps, parent=None): super().__init__(parent) self.setWindowTitle("Step Manager") self.resize(600, 400) - - main_layout = QVBoxLayout(self) - self.steps = steps - self.all_sequences = [ - s for s in self.steps if s.get("steptype") == "sequence_folder" - ] - + self.rows: list[dict[str, Any]] = [] + layout = QVBoxLayout(self) self.sequence_selector = QComboBox() - for seq in self.all_sequences: - name = seq["step_name"] - seq_id = seq["_sequence_id"] - self.sequence_selector.addItem(name, seq_id) - - self.sequence_selector.currentIndexChanged.connect(self.on_sequence_changed) - main_layout.addWidget(QLabel("Select Sequence:")) - main_layout.addWidget(self.sequence_selector) - - self.skip = QCheckBox("Skip All") - self.skip.setChecked(False) - - self.err = QCheckBox("Continue on All") - self.err.setChecked(False) - layout = QHBoxLayout() - layout.addWidget(self.skip) - layout.addWidget(self.err) - main_layout.addLayout(layout) - + self.sequences = [item for item in steps if item.get("steptype") == "sequence_folder"] + for sequence in self.sequences: + self.sequence_selector.addItem(sequence["step_name"], sequence["_sequence_id"]) + layout.addWidget(QLabel("Select sequence:")) + layout.addWidget(self.sequence_selector) + self.skip = QCheckBox("Skip all") + self.err = QCheckBox("Continue on error for all") + toggles = QHBoxLayout() + toggles.addWidget(self.skip) + toggles.addWidget(self.err) + layout.addLayout(toggles) scroll = QScrollArea() scroll.setWidgetResizable(True) - main_layout.addWidget(scroll) - container = QWidget() self.container_layout = QVBoxLayout(container) scroll.setWidget(container) - - self.rows = [] # list of dicts holding widgets for each step - self.on_sequence_changed(index=0) - self.container_layout.addStretch() - + layout.addWidget(scroll) buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - main_layout.addWidget(buttons) buttons.accepted.connect(self.accept) buttons.rejected.connect(self.reject) - - self.skip.stateChanged.connect(self.toggle_skip_all) - self.err.stateChanged.connect(self.toggle_continue_all) - - def build_row(self, step): - - node = step.get("_node", {}) - layout = QHBoxLayout() - #seq_name = self.get_sequence_name(step.get("_sequence_id")) - - name = QLineEdit(node.get("step_name", "")) - name.setDisabled(True) - - skip = QCheckBox("Skip") - skip.setChecked(node.get("skip", False)) - - err = QCheckBox("Continue on Error") - err.setChecked(node.get("continue_on_error", False)) - - #layout.addWidget(QLabel(f"Sequence: {seq_name}")) - layout.addWidget(QLabel("Name:")) - layout.addWidget(name) - layout.addWidget(skip) - layout.addWidget(err) - - return { - "layout": layout, - "step_data": step, - "name": name, - "skip": skip, - "err": err, - } - - def accept(self): - - for row in self.rows: - self.result_step = row["step_data"] - node = self.result_step.get("_node", {}) - - # Update only the skip/continue flags - node["skip"] = row["skip"].isChecked() - node["continue_on_error"] = row["err"].isChecked() - - super().accept() - - def get_steps(self): - return self.steps - - def toggle_skip_all(self, checked): - for row in self.rows: - row["skip"].setChecked(checked) - - def toggle_continue_all(self, checked): - for row in self.rows: - row["err"].setChecked(checked) - - def get_sequence_name(self, seq_id): - if not self.all_sequences: - return "Unknown" - - for seq in self.all_sequences: - if seq.get("_sequence_id") == seq_id: - return seq.get("step_name", "Unknown") - return "Unknown" - - def on_sequence_changed(self, index): - self.skip.setChecked(False) - self.err.setChecked(False) - seq_id = self.sequence_selector.itemData(index) - if seq_id is None: - return - self.load_sequence(seq_id) - - def load_steps(self, steps): - self.clear_container_layout() - self.rows.clear() - for step in steps: - row = self.build_row(step) - self.rows.append(row) - self.container_layout.addLayout(row["layout"]) - - def load_sequence(self, sequence_id): - seq = next((s for s in self.all_sequences - if s["_sequence_id"] == sequence_id), None) - if not seq: - return - - setup_folder = next((f for f in seq["children"] if f["steptype"] == "setup_folder"), None) - setup_steps = setup_folder["children"] if setup_folder else [] - main_folder = next((f for f in seq["children"] if f["steptype"] == "main_folder"), None) - main_steps = main_folder["children"] if main_folder else [] - teardown_folder = next((f for f in seq["children"] if f["steptype"] == "teardown_folder"), None) - teardown_steps = teardown_folder["children"] if teardown_folder else [] - steps = setup_steps + main_steps + teardown_steps - self.load_steps(steps) - - def clear_container_layout(self): + layout.addWidget(buttons) + self.sequence_selector.currentIndexChanged.connect(self._load_sequence) + self.skip.stateChanged.connect( + lambda checked: [row["skip"].setChecked(bool(checked)) for row in self.rows] + ) + self.err.stateChanged.connect( + lambda checked: [row["err"].setChecked(bool(checked)) for row in self.rows] + ) + self._load_sequence(0) + + def _clear_rows(self) -> None: while self.container_layout.count(): item = self.container_layout.takeAt(0) if item.widget(): item.widget().deleteLater() elif item.layout(): - self.clear_layout_recursive(item.layout()) - def clear_layout_recursive(self, layout): - while layout.count(): - item = layout.takeAt(0) - if item.widget(): - item.widget().deleteLater() - elif item.layout(): - self.clear_layout_recursive(item.layout()) - - + while item.layout().count(): + child = item.layout().takeAt(0) + if child.widget(): + child.widget().deleteLater() + self.rows.clear() + def _load_sequence(self, index: int) -> None: + self._clear_rows() + sequence_id = self.sequence_selector.itemData(index) + sequence = next( + (item for item in self.sequences if item.get("_sequence_id") == sequence_id), + None, + ) + if sequence is None: + return + for folder in sequence.get("children", []): + for step in folder.get("children", []): + node = step["_node"] + row_layout = QHBoxLayout() + row_layout.addWidget(QLabel(node.get("step_name", "Unnamed step"))) + skip = QCheckBox("Skip") + skip.setChecked(node.get("skip", False)) + error = QCheckBox("Continue on error") + error.setChecked(node.get("continue_on_error", False)) + row_layout.addWidget(skip) + row_layout.addWidget(error) + self.container_layout.addLayout(row_layout) + self.rows.append({"step": step, "skip": skip, "err": error}) + self.container_layout.addStretch() + def accept(self) -> None: + for row in self.rows: + row["step"]["_node"]["skip"] = row["skip"].isChecked() + row["step"]["_node"]["continue_on_error"] = row["err"].isChecked() + self.result_steps = self.steps + super().accept() class Step_setup(QDialog): - def __init__(self,use_input_mapping=True, use_output_mapping=True, parent=None): + """Create or edit any StepDefinition through the production JSON Schema.""" + + def __init__(self, use_input_mapping=True, use_output_mapping=True, parent=None): super().__init__(parent) self.setWindowTitle("New step creation") - self.resize(500,500) - layout = QVBoxLayout(self) + self.resize(500, 500) + self.AlreadyID = None + self.form_description = recipe_form_description() + self._skip_warning = False + self._previous_step_type = "" + self.schema_form: SchemaFormWidget | None = None - layout.addWidget(QLabel("Steptype")) + layout = QVBoxLayout(self) + layout.addWidget(QLabel("Step type")) self.list_steptype = QComboBox() - self.steptypes = list(discriminator_schemas("StepDefinition")) - self.list_steptype.addItems(self.steptypes) + self.list_steptype.addItems(self.form_description["steps"]) layout.addWidget(self.list_steptype) - - self.list_steptype.currentTextChanged.connect(self.on_step_type_changed) - self._previous_step_type = self.list_steptype.currentText() - self._skip_warning = False - self.form_description = recipe_form_description() - self.schema_form = None - - # Container for step-specific widgets self.step_specific_container = QVBoxLayout() - container_widget = QWidget() - container_widget.setLayout(self.step_specific_container) - self._render_schema_step(self.list_steptype.currentText()) - + container = QWidget() + container.setLayout(self.step_specific_container) scroll = QScrollArea() scroll.setWidgetResizable(True) - scroll.setWidget(container_widget) + scroll.setWidget(container) layout.addWidget(scroll) - - # OK/Cancel buttons buttons = QDialogButtonBox() self.ok_button = QPushButton("OK") self.cancel_button = QPushButton("Cancel") - self.ok_button.setStyleSheet("color: black;") - self.cancel_button.setStyleSheet("color: black;") buttons.addButton(self.ok_button, QDialogButtonBox.AcceptRole) buttons.addButton(self.cancel_button, QDialogButtonBox.RejectRole) - + layout.addWidget(buttons) self.ok_button.clicked.connect(self.accept) self.cancel_button.clicked.connect(self.reject) - layout.addWidget(buttons) - self.AlreadyID = None - - def on_step_type_changed(self, step_type: str): - if not getattr(self, "_skip_warning", False): - msg = QMessageBox() - msg.setIcon(QMessageBox.Warning) - msg.setWindowTitle("Warning") - msg.setText( - "Changing this step type will delete all current information.\n\n" - "Do you want to continue?" - ) - msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel) - msg.setDefaultButton(QMessageBox.Cancel) - - checkbox = QCheckBox("Don't ask again") - msg.setCheckBox(checkbox) - - result = msg.exec() - if checkbox.isChecked(): - self._skip_warning = True + initial_type = self.list_steptype.currentText() + self._render_schema_step(initial_type) + self._previous_step_type = initial_type + self.list_steptype.currentTextChanged.connect(self.on_step_type_changed) - if result == QMessageBox.Cancel: - # User canceled → revert selection and stop everything + def _current_values_for_switch(self) -> dict[str, Any]: + if self.schema_form is None: + return {} + try: + return self.schema_form.values() + except (TypeError, ValueError): + return {} + + def on_step_type_changed(self, step_type: str) -> None: + previous_values = self._current_values_for_switch() + if self._previous_step_type and not self._skip_warning: + answer = QMessageBox.question( + self, + "Change step type", + "Compatible common fields will be retained; fields specific to the old " + "type will be removed. Continue?", + QMessageBox.Ok | QMessageBox.Cancel, + QMessageBox.Cancel, + ) + if answer == QMessageBox.Cancel: self.list_steptype.blockSignals(True) self.list_steptype.setCurrentText(self._previous_step_type) self.list_steptype.blockSignals(False) return self._previous_step_type = step_type - # Clear previous widgets - self.clear_layout(self.step_specific_container) - self._render_schema_step(step_type) + self._render_schema_step(step_type, previous_values) - def _render_schema_step(self, step_type: str): - """Render the selected step definition directly from JSON Schema.""" + def _render_schema_step( + self, step_type: str, retained_values: dict[str, Any] | None = None + ) -> None: + self._clear_layout(self.step_specific_container) self.schema_form = SchemaFormWidget(self.form_description["steps"][step_type]) self.step_specific_container.addWidget(self.schema_form) self.step_specific_container.addStretch() + if retained_values: + allowed = self.form_description["steps"][step_type]["fields"] + self.schema_form.load_values( + {name: value for name, value in retained_values.items() if name in allowed} + ) def load_definition(self, node: dict[str, Any]) -> None: - """Load an existing canonical step into the schema-driven editor.""" + """Load an existing canonical step into the same form used for new steps.""" step_type = node["steptype"] + if step_type not in self.form_description["steps"]: + raise ValueError(f"Unsupported canonical step type: {step_type}") self._skip_warning = True self.list_steptype.setCurrentText(step_type) self._skip_warning = False + self._previous_step_type = step_type + assert self.schema_form is not None self.schema_form.load_values(node) self.setWindowTitle(f"Edit Step: {node.get('step_name', step_type)}") - - def clear_layout(self, layout): + + @staticmethod + def _clear_layout(layout) -> None: while layout.count(): item = layout.takeAt(0) - # Remove widgets if item.widget(): item.widget().deleteLater() - # Remove nested layouts elif item.layout(): - self.clear_layout(item.layout()) - - # --------- Setup functions for each steptype --------- - def setup_pythonmodulestep(self): - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New step") - self.step_specific_container.addWidget(QLabel("Step name")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.list_actiontypes = QComboBox() - self.action_types = ["method", "read_attribute", "write_attribute"] - self.list_actiontypes.addItems(self.action_types) - self.step_specific_container.addWidget(QLabel("Action Type")) - self.step_specific_container.addWidget(self.list_actiontypes) - - self.module_input = QLineEdit() - self.module_input.setText("example_tests") - self.step_specific_container.addWidget(QLabel("Specify filename for the script")) - self.step_specific_container.addWidget(self.module_input) - - self.method_input = QLineEdit() - self.method_input.setText("main") - self.step_specific_container.addWidget(QLabel("Name of method to run")) - self.step_specific_container.addWidget(self.method_input) - - self.step_specific_container.addWidget(QLabel("Input Mapping")) - self.input_mapping_widget = self.InOutputMappingWidget() - self.step_specific_container.addWidget(self.input_mapping_widget) - - self.step_specific_container.addWidget(QLabel("Output Mapping")) - self.output_mapping_widget = self.InOutputMappingWidget( output = True) - self.step_specific_container.addWidget(self.output_mapping_widget) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - def setup_userinteractionstep(self): - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New UserInteractionStep") - self.step_specific_container.addWidget(QLabel("Step name")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.step_specific_container.addWidget(QLabel("Input Mapping")) - self.input_mapping_widget = self.InOutputMappingWidget(allow_message=True, allow_image=True, allow_options=True, no_extraSteps=True) - self.step_specific_container.addWidget(self.input_mapping_widget) - - self.step_specific_container.addWidget(QLabel("Output Mapping")) - self.output_mapping_widget = self.InOutputMappingWidget( output = True, no_extraSteps=True, specific_method="output") - self.step_specific_container.addWidget(self.output_mapping_widget) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - - def setup_waitstep(self): - - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New waitstep") - self.step_specific_container.addWidget(QLabel("Stepname")) - self.step_specific_container.addWidget(self.step_name_input) - - self.wait_time_input = QLineEdit() - self.wait_time_input.setText("3") - self.step_specific_container.addWidget(QLabel("Wait time (seconds)")) - self.step_specific_container.addWidget(self.wait_time_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(100) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - - def setup_userloadingstep(self): - - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New UserLoadingStep") - self.step_specific_container.addWidget(QLabel("Step name")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.step_specific_container.addWidget(QLabel("Input Mapping")) - self.input_mapping_widget = self.InOutputMappingWidget(allow_message=True, allow_image=True,allow_options=True, no_extraSteps=True) - self.input_mapping_widget.add_option_row(key="cancel", value="cancel") - self.input_mapping_widget.add_option_row(key="file", value="ButtonKey for fileloading") - self.input_mapping_widget.add_option_row() - self.step_specific_container.addWidget(self.input_mapping_widget) - - - self.step_specific_container.addWidget(QLabel("Output Mapping")) - self.output_mapping_widget = self.InOutputMappingWidget(loader=True, no_extraSteps=False, specific_method="passfail") - self.step_specific_container.addWidget(self.output_mapping_widget) - - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - def setup_userrunmethodstep(self): - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New Userstep") - self.step_specific_container.addWidget(QLabel("Step name")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.list_actiontypes = QComboBox() - self.action_types = ["method", "read_attribute", "write_attribute"] - self.list_actiontypes.addItems(self.action_types) - self.step_specific_container.addWidget(QLabel("Action Type")) - self.step_specific_container.addWidget(self.list_actiontypes) - - self.module_input = QLineEdit() - self.module_input.setText("example_tests") - self.step_specific_container.addWidget(QLabel("Specify filename for the script")) - self.step_specific_container.addWidget(self.module_input) - - self.step_specific_container.addWidget(QLabel("Input Mapping")) - layout = QHBoxLayout() - layout.addWidget(QLabel(" Trigger response:")) - self.trigger_response = QLineEdit() - self.trigger_response.setPlaceholderText("Name the value of the key for the options button desired to trigger method") - layout.addWidget(self.trigger_response) - self.step_specific_container.addLayout(layout) - self.input_mapping_widget = self.InOutputMappingWidget(allow_message=True, allow_image=True, allow_method=True, allow_options=True) - self.step_specific_container.addWidget(self.input_mapping_widget) - - self.step_specific_container.addWidget(QLabel("Output Mapping")) - self.output_mapping_widget = self.InOutputMappingWidget( output = True) - self.step_specific_container.addWidget(self.output_mapping_widget) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - def setup_userwritestep(self): - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New UserWriteStep") - self.step_specific_container.addWidget(QLabel("Step name")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.step_specific_container.addWidget(QLabel("Input Mapping")) - self.input_mapping_widget = self.InOutputMappingWidget(allow_message=True, allow_image=True, no_extraSteps=True) - self.step_specific_container.addWidget(self.input_mapping_widget) - - self.step_specific_container.addWidget(QLabel("Determine type of function")) - self.UART_setup = QCheckBox("UART") - self.Write_variable = QCheckBox("Write to variable") - mode_choice = QHBoxLayout() - mode_choice.addWidget(self.UART_setup) - mode_choice.addWidget(self.Write_variable) - self.chosen_input = QLineEdit() - self.chosen_input.setPlaceholderText("Explaination of what will happen once either of the above are chosen. ") - self.chosen_input.setDisabled(True) - self.step_specific_container.addLayout(mode_choice) - self.step_specific_container.addWidget(self.chosen_input) - self.UART_setup.stateChanged.connect(self.Uart_Toggle) - self.Write_variable.stateChanged.connect(self.Write_toggle) + Step_setup._clear_layout(item.layout()) - self.step_specific_container.addWidget(QLabel("Output Mapping")) - self.output_mapping_widget = self.InOutputMappingWidget(loader=True, no_extraSteps=False) - self.step_specific_container.addWidget(self.output_mapping_widget) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - self.step_specific_container.addLayout(checkbox_horizontal) - self.step_specific_container.addStretch() - - def setup_sshconnectstep(self): - - if hasattr(self, "input_mapping_widget"): - self.input_mapping_widget.setParent(None) - self.input_mapping_widget.deleteLater() - del self.input_mapping_widget - - if hasattr(self, "output_mapping_widget"): - self.output_mapping_widget.setParent(None) - self.output_mapping_widget.deleteLater() - del self.output_mapping_widget - - - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New SSH Close step") - self.step_specific_container.addWidget(QLabel("Stepname")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.step_specific_container.addWidget(QLabel("Global variables used for SSH")) - self.SSHInfo_exists = QCheckBox("Already exists?") - self.SSHInfo_exists.setChecked(False) - self.step_specific_container.addWidget(self.SSHInfo_exists) - - self.GLB_host = QLineEdit() - self.GLB_host.setPlaceholderText("Hostname, either name or IP") - self.GLB_user = QLineEdit() - self.GLB_user.setPlaceholderText("root") - self.GLB_passwd = QLineEdit() - self.GLB_passwd.setPlaceholderText("1234") - self.GLB_pkey = QLineEdit() - self.GLB_pkey.setPlaceholderText("Path to key") - self.GLB_port = QLineEdit() - self.GLB_port.setPlaceholderText("Only change if non default port is used") - - def add_row(label_text, widget): - layout = QHBoxLayout() - layout.addWidget(QLabel(label_text)) - layout.addWidget(widget) - self.step_specific_container.addLayout(layout) - - add_row("Host", self.GLB_host) - add_row("Username", self.GLB_user) - add_row("Password", self.GLB_passwd) - add_row("Private Key location", self.GLB_pkey) - add_row("Port", self.GLB_port) - - def toggle_ssh_fields(state): - enabled = not bool(state) # if checkbox is checked, disable fields - for widget in [self.GLB_host, self.GLB_user, self.GLB_passwd, self.GLB_pkey, self.GLB_port]: - widget.setEnabled(enabled) - - self.SSHInfo_exists.stateChanged.connect(toggle_ssh_fields) - - self.skip_checkbox = QCheckBox("Skip") - self.skip_checkbox.setChecked(True) - self.continue_on_error_checkbox = QCheckBox("Continue on Error") - self.continue_on_error_checkbox.setChecked(True) - - checkbox_horizontal = QHBoxLayout() - checkbox_horizontal.addWidget(self.skip_checkbox) - checkbox_horizontal.addWidget(self.continue_on_error_checkbox) - - self.step_specific_container.addLayout(checkbox_horizontal) - - self.step_specific_container.addStretch() - - def setup_sshclosestep(self): - self.step_name_input = QLineEdit() - self.step_name_input.setPlaceholderText("New SSH Close step") - self.step_specific_container.addWidget(QLabel("Stepname")) - self.step_specific_container.addWidget(self.step_name_input) - - self.description_input = QTextEdit() - self.description_input.setMaximumHeight(70) - self.description_input.setPlaceholderText("describe the test in this box") - self.step_specific_container.addWidget(QLabel("Description")) - self.step_specific_container.addWidget(self.description_input) - - self.step_specific_container.addStretch() - - - - def accept(self): + def accept(self) -> None: + assert self.schema_form is not None step_type = self.list_steptype.currentText() - if self.AlreadyID: - StepID = self.AlreadyID - else: - StepID = str(uuid.uuid4()) - try: - step_definition_data = self.schema_form.values() - step_definition_data["steptype"] = step_type - definition = TypeAdapter(StepDefinition).validate_python( - step_definition_data - ) - except Exception as error: - self.setStyleSheet("""QMessageBox QPushButton { color: black;}""") + data = self.schema_form.values() + data["steptype"] = step_type + definition = TypeAdapter(StepDefinition).validate_python(data) + except (TypeError, ValueError, ValidationError) as error: QMessageBox.warning(self, "Invalid step", str(error)) return @@ -965,894 +563,7 @@ def accept(self): self.result_step = { "steptype": step_type, "step_name": definition.step_name, - "_parent": "main_folder", "_node": node, - "_id": StepID, + "_id": self.AlreadyID or str(uuid.uuid4()), } - self.global_variables = {} - self.local_variables = {} - globals_in, locals_in = self.extract_locals_globals(node["input_mapping"]) - globals_out, locals_out = self.extract_locals_globals(node["output_mapping"]) - self.global_variables.update(globals_in) - self.global_variables.update(globals_out) - self.local_variables.update(locals_in) - self.local_variables.update(locals_out) super().accept() - return - - validate_method = f"validate_{step_type.lower()}" - if hasattr(self, validate_method): - ok, error_msg = getattr(self, validate_method)() - if not ok: - self.setStyleSheet("""QMessageBox QPushButton { color: black;}""") - msg = QMessageBox.warning(self,"Missing data", error_msg) - return - - self.result_step = { - "steptype": step_type, - "step_name": "Default", - "_parent": None, - "_node": {}, - "_id": StepID - } - self.global_variables = {} - self.local_variables = {} - g_in, l_in = None, None - g_out, l_out = None,None - match step_type: - case "PythonModuleStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "PythonModuleStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "action_type": self.list_actiontypes.currentText(), - "module": self.module_input.text(), - "method_name": self.method_input.text(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "input_mapping": self.input_mapping_widget.get_data(), - "output_mapping": self.output_mapping_widget.get_data() - } - g_in, l_in = self.extract_locals_globals(self.input_mapping_widget.get_data()) - g_out, l_out = self.extract_locals_globals(self.output_mapping_widget.get_data()) - - case "UserInteractionStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "UserInteractionStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "input_mapping": self.input_mapping_widget.get_data(), - "output_mapping": self.output_mapping_widget.get_data() - } - g_in, l_in = self.extract_locals_globals(self.input_mapping_widget.get_data()) - g_out, l_out = self.extract_locals_globals(self.output_mapping_widget.get_data()) - - case "WaitStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "WaitStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "input_mapping": {"wait_time": {"value": self.wait_time_input.text()}}, - "output_mapping": {} - } - - case "UserLoadingStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "UserInteractionStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "input_mapping": self.input_mapping_widget.get_data(), - "output_mapping": self.output_mapping_widget.get_data() - } - self.global_variables = { - "loadFile": 'file' - } - g_in, l_in = self.extract_locals_globals(self.input_mapping_widget.get_data()) - g_out, l_out = self.extract_locals_globals(self.output_mapping_widget.get_data()) - - case "UserRunMethodStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "UserRunMethodStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "action_type": self.list_actiontypes.currentText(), - "module": self.module_input.text(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "trigger_response": self.trigger_response.text(), - "input_mapping": self.input_mapping_widget.get_data(), - "output_mapping": self.output_mapping_widget.get_data() - } - g_in, l_in = self.extract_locals_globals(self.input_mapping_widget.get_data()) - g_out, l_out = self.extract_locals_globals(self.output_mapping_widget.get_data()) - case "UserWriteStep": - self.input_mapping_widget.add_special_row("options") - self.input_mapping_widget.add_option_row(key="cancel", value="cancel") - if self.Write_variable.isChecked(): - self.input_mapping_widget.add_option_row(key="wrt", value="Write to variable") - elif self.UART_setup.isChecked(): - self.input_mapping_widget.add_option_row(key="ID", value="Setup UART") - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "UserWriteStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - "input_mapping": self.input_mapping_widget.get_data(), - "output_mapping": self.output_mapping_widget.get_data() - } - if self.UART_setup.isChecked(): - self.result_step["_node"]["output_mapping"] = {"output":{"type":"passfail"}} - self.global_variables = { - "cancel_key": 'cancel', - "ID_key": 'ID', - "wrt_key": 'wrt', - } - g_in, l_in = self.extract_locals_globals(self.input_mapping_widget.get_data()) - g_out, l_out = self.extract_locals_globals(self.output_mapping_widget.get_data()) - - case "SSHConnectStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "setup_folder" - self.result_step["_node"] = { - "steptype": "SSHConnectStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText(), - "skip": self.skip_checkbox.isChecked(), - "continue_on_error": self.continue_on_error_checkbox.isChecked(), - } - self.global_variables = { - "host": self.GLB_host.text(), - "user": self.GLB_user.text(), - "password": self.GLB_passwd.text(), - "private_key": self.GLB_pkey.text(), - "port": self.GLB_port.text() if self.GLB_port.text() != "" else "none" - } - - case "SSHCloseStep": - self.result_step["step_name"] = self.step_name_input.text() - self.result_step["_parent"] = "main_folder" - self.result_step["_node"] = { - "steptype": "SSHCloseStep", - "step_name": self.result_step["step_name"], - "description": self.description_input.toPlainText() - } - case _: - print("Unknown step type") - if g_in and g_out and l_in and l_out is not None: - self.global_variables.update(g_in) - self.global_variables.update(g_out) - self.local_variables.update(l_in) - self.local_variables.update(l_out) - - super().accept() - - def load_existing_globals(self, data): - """Load saved SSH global variables into the existing input fields.""" - - # Map data keys to the actual widgets created in setup_sshconnectstep() - widget_map = { - "host": self.GLB_host, - "user": self.GLB_user, - "password": self.GLB_passwd, - "private_key": self.GLB_pkey, - "port": self.GLB_port - } - - for key, widget in widget_map.items(): - if key in data: - value = data[key] - widget.setText("" if value is None else str(value)) - self.SSHInfo_exists.setChecked(True) - self.SSHInfo_exists.stateChanged.emit(True) - - ################ Validation steps for the settings ################################# - def validate_pythonmodulestep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - # Validate input mapping - ok, msg = self.input_mapping_widget.validate() - if not ok: - return False, msg - - # Validate output mapping - ok, msg = self.output_mapping_widget.validate() - if not ok: - return False, msg - - return True, "" - - def validate_userinteractionstep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - # Validate input mapping - ok, msg = self.input_mapping_widget.validate() - if not ok: - if not "Image" in msg: - return False, msg - - - # Validate output mapping - ok, msg = self.output_mapping_widget.validate() - if not ok: - return False, msg - - return True, "" - - def validate_waitstep(self): - if not self.wait_time_input.text().strip(): - return False, "Please enter a wait time." - if not self.wait_time_input.text().isdigit(): - return False, "Wait time must be a number." - if not self.step_name_input.text(): - return False, "Step must have a name" - return True, "" - - def validate_userloadingstep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - # Validate input mapping - ok, msg = self.input_mapping_widget.validate() - if not ok: - if not "Image" in msg: - return False, msg - - - table = self.input_mapping_widget.get_options_table() - has_valid = False - for r in range(table.rowCount()): - key_item = table.item(r, 0) - if key_item and key_item.text().strip()=="file": - has_valid = True - if not has_valid: - return False, f"'file' must exist as a key" - - - # Validate output mapping - ok, msg = self.output_mapping_widget.validate() - if not ok: - return False, msg - - return True, "" - - def validate_userrunmethodstep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - # Validate input mapping - ok, msg = self.input_mapping_widget.validate() - if not ok: - return False, msg - - # Validate output mapping - ok, msg = self.output_mapping_widget.validate() - if not ok: - return False, msg - - return True, "" - - def validate_userwritestep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - if not self.UART_setup.isChecked() and not self.Write_variable.isChecked(): - return False, "You must choose which functionType to use" - - # Validate input mapping - ok, msg = self.input_mapping_widget.validate() - if not ok: - if not "Image" in msg: - return False, msg - - # Validate output mapping - ok, msg = self.output_mapping_widget.validate() - if not ok: - return False, msg - - return True, "" - - - def validate_sshconnectstep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - return True, "" - - def validate_sshclosestep(self): - if not self.step_name_input.text().strip(): - return False, "Step must have a name" - - return True, "" - - def Write_toggle(self): - if self.Write_variable.isChecked(): - self.UART_setup.setChecked(False) - self.chosen_input.setText("Chosing this will allow the function to write to a specific output") - - def Uart_Toggle(self): - if self.UART_setup.isChecked(): - self.Write_variable.setChecked(False) - self.chosen_input.setText("Chosing this will make the step setup a UART that can be used for other tests. Choosing this will bypass the output mapping") - - def extract_locals_globals(self, mapping): - IGNORED = {"message", "options", "image_path", "method"} - - globals_found = {} - locals_found = {} - - for key, entry in mapping.items(): - if key in IGNORED: - continue - - if not isinstance(entry, dict): - continue - - var_type = entry.get("type") - value = entry.get("value") - if var_type == "global": - value = entry.get("global_name") - globals_found[value] = "" - elif var_type == "local": - value = entry.get("local_name") - locals_found[value] = "" - - return globals_found, locals_found - - - class InOutputMappingWidget(QWidget): - def __init__(self, parent=None, output= False, allow_image = False, allow_options=False,allow_message=False, allow_method = False, no_extraSteps=False, specific_method = None, loader = None): - super().__init__(parent) - self.Output = output - self.allow_image = allow_image - self.allow_options = allow_options - self.allow_message = allow_message - self.allow_method = allow_method - self.no_extraSteps = no_extraSteps - self.specific_method = specific_method - self.loader = loader - - self.layout = QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(2) - self.rows = [] - - self.populate_required_rows() - if not self.no_extraSteps: - self.add_row() - - # ---------------- Pre-populate required rows ---------------- - def populate_required_rows(self): - if self.allow_message: - self.add_special_row("message") - if self.allow_options: - self.add_special_row("options") - if self.allow_image: - self.add_special_row("image_path") - if self.allow_method: - self.add_special_row("method") - if self.specific_method: - self.add_special_row(self.specific_method) - - def add_special_row(self, type_name): - row = {} - - - container = QWidget() - h = QHBoxLayout(container) - h.setContentsMargins(0, 0, 0, 0) - h.setSpacing(2) - - # Parameter name - row["name_edit"] = QLineEdit() - row["name_edit"].setPlaceholderText(type_name) - row["name_edit"].setReadOnly(True) - h.addWidget(row["name_edit"]) - - # Type combobox - row["type_combo"] = QComboBox() - row["type_combo"].addItem(type_name) - row["type_combo"].setCurrentText(type_name) - row["type_combo"].setEnabled(False) - h.addWidget(row["type_combo"]) - - # Value edit - row["value_edit"] = QLineEdit() - row["value_edit"].setPlaceholderText("fill in value") - h.addWidget(row["value_edit"]) - - # Second value for ranges (hidden) - row["value_edit2"] = QLineEdit() - row["value_edit2"].setVisible(False) - h.addWidget(row["value_edit2"]) - - row["special"] = QLineEdit() - row["special"].setText(type_name) - row["special"].setVisible(False) - h.addWidget(row["special"]) - - # File button or options table - row["file_button"] = QPushButton("📁") - row["file_button"].setVisible(False) - row["options_widget"] = None - if type_name == "image_path": - row["file_button"].setVisible(True) - row["file_button"].clicked.connect(lambda _, r=row: self.pick_file(r)) - h.addWidget(row["file_button"]) - - self.layout.addWidget(container) - self.rows.append(row) - self.on_type_changed(row) - - row["name_edit"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - row["value_edit"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - row["value_edit2"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - - - # ---------------- Dynamic row changes ---------------- - def add_row(self): - row = {} - - container = QWidget() - h = QHBoxLayout(container) - h.setContentsMargins(0, 0, 0, 0) - h.setSpacing(2) - - # Parameter name - row["name_edit"] = QLineEdit() - if self.Output: - row["name_edit"].setPlaceholderText("parameter (Set the name to be the input of your method)") - elif self.loader: - row["name_edit"].setPlaceholderText("Specify variable to save the loaded file/element") - row["name_edit"].setVisible - else: - row["name_edit"].setPlaceholderText("parameter (e.g. value)") - row["name_edit"].textChanged.connect(self.on_edit_changed) - h.addWidget(row["name_edit"]) - - # direct/global/local - row["type_combo"] = QComboBox() - mapping_name = "OutputMapping" if self.Output else "InputMapping" - types = list(discriminator_schemas(mapping_name)) - - row["type_combo"].addItems(types) - - h.addWidget(row["type_combo"]) - row["type_combo"].currentTextChanged.connect(lambda _, r=row: self.on_type_changed(r)) - - # First value input - row["value_edit"] = QLineEdit() - row["value_edit"].setPlaceholderText("value or variable name") - row["value_edit"].textChanged.connect(self.on_edit_changed) - h.addWidget(row["value_edit"]) - - # Second value (only visible in "range") - row["value_edit2"] = QLineEdit() - row["value_edit2"].setPlaceholderText("max") - row["value_edit2"].setVisible(False) - h.addWidget(row["value_edit2"]) - - - row["file_button"] = QPushButton("📁") - row["file_button"].setVisible(False) - row["file_button"].clicked.connect(lambda checked, r=row: self.pick_file(r)) - h.addWidget(row["file_button"]) - - # options editor (sub-table) - row["options_widget"] = None - - self.layout.addWidget(container) - self.rows.append(row) - - #self.on_type_changed(row) - - row["name_edit"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - row["value_edit"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - row["value_edit2"].editingFinished.connect(lambda r=row: self.on_edit_finished(r)) - - def del_rows(self): - def is_blank(row): - """Return True if the row has no meaningful data.""" - - name = row["name_edit"].text().strip() - value = row["value_edit"].text().strip() - value2 = row["value_edit2"].text().strip() if row.get("value_edit2") else "" - if row.get("options_widget"): - table = row["options_widget"] - for r in range(table.rowCount()): - key_item = table.item(r, 0) - val_item = table.item(r, 1) - if (key_item and key_item.text().strip()) or (val_item and val_item.text().strip()): - return False - return not (name or value or value2) - - new_rows = [] - last_non_blank_index = -1 - - # Find the last row that has content - for i, row in enumerate(self.rows): - if not is_blank(row): - last_non_blank_index = i - - # Rebuild rows, keeping only meaningful blanks - for i, row in enumerate(self.rows): - blank = is_blank(row) - if blank and i < last_non_blank_index: - # Remove middle blank rows - if not row["special"]: - widget = row["name_edit"].parentWidget() - self.layout.removeWidget(widget) - widget.setParent(None) - widget.deleteLater() - else: - new_rows.append(row) - - # Ensure at least one blank row at the end - if not new_rows or not is_blank(new_rows[-1]) and not self.no_extraSteps: - self.add_row() - self.rows = new_rows - - def on_edit_finished(self, row): - """Called when a row loses focus.""" - self.del_rows() - - def pick_file(self, row): - """Choose a file for image_path""" - import shutil - file_path, _ = QFileDialog.getOpenFileName(self, "Select Image", "", "Images (*.png *.jpg *.jpeg)") - - if file_path: - target_folder = "./src/pypts/images/" - os.makedirs(target_folder, exist_ok=True) - - filename = os.path.basename(file_path) - destination = os.path.join(target_folder, filename) - if not os.path.exists(destination): - shutil.copy(file_path, destination) - - row["value_edit"].setText(filename) - - def make_options_editor(self, row): - table = QTableWidget(1, 2) # 1 row, 2 columns - table.setHorizontalHeaderLabels(["Key", "Value"]) - table.horizontalHeader().setStretchLastSection(True) - table.verticalHeader().setVisible(False) - - table.setStyleSheet(""" - QTableWidget::item { - padding: 0px; - } - QHeaderView::section { - padding: 0px; - font-size: 12px; - } - """) - - table.verticalHeader().setDefaultSectionSize(18) - - table.setFixedWidth(260) - table.setFixedHeight(180) - - table.setItem(0, 0, QTableWidgetItem("")) - table.setItem(0, 1, QTableWidgetItem("")) - - table.setEditTriggers(QAbstractItemView.AllEditTriggers) - - # Store reference - row["options_widget"] = table - - self.layout.addWidget(table) - - # Add a new blank row when last row has text - def on_cell_changed(_row, _col): - last_row = table.rowCount() - 1 - key = table.item(last_row, 0) - val = table.item(last_row, 1) - if key and key.text().strip() or val and val.text().strip(): - table.insertRow(table.rowCount()) - table.setItem(table.rowCount() - 1, 0, QTableWidgetItem("")) - table.setItem(table.rowCount() - 1, 1, QTableWidgetItem("")) - - self.on_edit_changed() - - table.cellChanged.connect(on_cell_changed) - - def get_options_table(self): - for r in self.rows: - if r.get("options_widget"): - return r["options_widget"] - return None - - def add_option_row(self, key="", value=""): - """Append a row to the options table and set key/value (safe vs cellChanged).""" - table = self.get_options_table() - if table is None: - return - table.blockSignals(True) - try: - row_index = table.rowCount() - table.insertRow(row_index) - table.setItem(row_index, 0, QTableWidgetItem(key)) - table.setItem(row_index, 1, QTableWidgetItem(value)) - finally: - table.blockSignals(False) - try: - if hasattr(self, "on_edit_changed"): - self.on_edit_changed() - except Exception: - pass - - def on_type_changed(self, row): - t = row["type_combo"].currentText() - - # Reset everything - row["value_edit"].setVisible(True) - row["value_edit2"].setVisible(False) - row["file_button"].setVisible(False) - if row.get("options_widget"): - row["name_edit"].setVisible(False) - if t == "range": - row["value_edit"].setPlaceholderText("min") - row["value_edit2"].setVisible(True) - elif t == "passthrough" or t == "passfail": - row["value_edit"].setVisible(False) - elif t == "image_path": - row["name_edit"].setVisible(False) - row["file_button"].setVisible(True) - row["value_edit"].setPlaceholderText("image file path") - elif t == "message": - row["name_edit"].setText("Deletionsaving") - row["name_edit"].setVisible(False) - elif t == "method": - #row["value_edit"].setText("Deletionsaving") - row["name_edit"].setVisible(False) - elif t == "options": - row["name_edit"].setVisible(False) - row["value_edit"].setVisible(False) - self.make_options_editor(row) - if self.specific_method: - row["name_edit"].setText("output") - row["name_edit"].setVisible(False) - - def on_edit_changed(self): - last = self.rows[-1] - has_text = ( - last["name_edit"].text().strip() - or last["value_edit"].text().strip() - or last["value_edit2"].text().strip() - ) - if last.get("options_widget"): - table = last["options_widget"] - for r in range(table.rowCount()): - key_item = table.item(r, 0) - val_item = table.item(r, 1) - if key_item and key_item.text().strip(): - has_text = True - if val_item and val_item.text().strip(): - has_text = True - if has_text: - self.add_row() - - - - def get_data(self): - """ - Build YAML-style input mapping, skipping empty rows. - """ - result = {} - for row in self.rows: - name = row["name_edit"].text().strip() - value1 = row["value_edit"].text().strip() - value2 = row["value_edit2"].text().strip() - type_ = row["type_combo"].currentText() - - - if not name and type_ not in {"message", "options", "image_path", "method", "output"}: - continue - - if type_ == "image_path": - result[type_] = {"type": "direct", "value": value1} - elif type_ == "options": - opts = [] - table = row.get("options_widget") - if table: - for r in range(table.rowCount()): - key_item = table.item(r, 0) - val_item = table.item(r, 1) - if key_item and val_item: - key = key_item.text().strip() - val = val_item.text().strip() - if key and val: - if key.lower() in {"yes", "no", "true", "false"}: - key = f"'{key}'" - if val.lower() in {"yes", "no", "true", "false"}: - val = f"'{val}'" - opts.append({key: val}) - result[type_] = {"type": "direct", "value": opts} - elif type_ == "message": - result[type_] = {"type": "direct", "value": value1} - - elif type_ == "method": - result[type_] = {"type": "method", "value": value1} - - elif type_ == "range": - if not (value1 and value2): - continue # skip incomplete - result[name] = {"type": "range", "min": value1, "max": value2} - - elif type_ == "local": - result[name] = {"type": "local", "local_name": value1} - - elif type_ == "global": - result[name] = {"type": "global", "global_name": value1} - - elif type_ == "passthrough": - result[name] = {"type": "passthrough"} - elif type_ == "equals": - result[name] = {"type": "equals", "value": value1} - elif type_ == "output": - result[type_] = {"type": "equals", "value": value1} - elif type_ == "passfail": - result[name] = {"type": "passfail"} - else: # direct - result[name] = {"type": "direct", "value": value1} - - return result - - - def validate(self): - found_message = False - found_options = False - found_image = False - found_method = False - - for row in self.rows: - - name = row["name_edit"].text().strip() - - type_ = row["type_combo"].currentText() - value1 = row["value_edit"].text().strip() - value2 = row["value_edit2"].text().strip() - - if not name and type_ not in {"message", "options", "image_path", "method"}: - continue # ignore blank row - - if type_ in ["direct", "global", "local", "equals", "method"] and not value1: - return False, f"Parameter '{name}' needs a value." - - if type_ == "range" and (not value1 or not value2): - return False, f"Parameter '{name}' needs min and max." - - - if type_ == "options": - table = row["options_widget"] - has_valid = False - for r in range(table.rowCount()): - key_item = table.item(r, 0) - val_item = table.item(r, 1) - if key_item and val_item and key_item.text().strip() and val_item.text().strip(): - has_valid = True - if not has_valid: - return False, f"Options for '{name}' must contain at least one key/value pair." - if type_ == "message": - found_message = True - if type_ == "options": - found_options = True - if type_ == "image_path": - found_image = True - if type_ == "method": - found_method = True - - # ---- REQUIRED TYPES CHECK ---- - if self.allow_message and not found_message: - return False, "A 'message' mapping is required." - if self.allow_options and not found_options: - return False, "At least one 'options' mapping is required." - if self.allow_image and not found_image: - return False, "An 'image_path' mapping is required." - if self.allow_method and not found_method: - return False, "A 'method' mapping is required." - return True, "" - - - def load_existing_data(self, data): - - self.clear_gui() - - def safe_set_text(widget, value): - widget.setText("" if value is None else str(value)) - added_row = False - - for name, info in data.items(): - - # Add a row for each mapping - special_types = {"output", "message", "options", "image_path", "method"} - is_special = name in special_types - - # Create a row - if is_special: - self.add_special_row(type_name=name) - row = self.rows[-1] - - if name == "options": - for opt in info.get("value", []): - for k, v in opt.items(): - table = row["options_widget"] - r = table.rowCount() - 1 - table.setItem(r, 0, QTableWidgetItem(k)) - table.setItem(r, 1, QTableWidgetItem(v)) - table.setItem(table.rowCount() - 1, 0, QTableWidgetItem("")) - table.setItem(table.rowCount() - 1, 1, QTableWidgetItem("")) - - elif name in ("image_path", "message", "output", "method","equals"): - safe_set_text(row["value_edit"], info.get("value")) - else: # direct / custom name - safe_set_text(row["value_edit"], info.get("value")) - if not is_special: - row["name_edit"].setVisible(True) - else: - row["name_edit"].setVisible(False) - - else: - type_ = info.get("type", "direct") - self.add_row() - row = self.rows[-1] - row["name_edit"].setText(name) - row["type_combo"].setCurrentText(type_) - # handle specific types - if type_ == "range": - safe_set_text(row["value_edit"], info.get("min", "")) - safe_set_text(row["value_edit2"], info.get("max", "")) - row["value_edit2"].setVisible(True) - - elif type_ == "local": - safe_set_text(row["value_edit"], info.get("local_name")) - elif type_ == "global": - safe_set_text(row["value_edit"], info.get("global_name")) - elif type_ in {"equals", "direct", "method"}: - safe_set_text(row["value_edit"], info.get("value")) - elif type_ == "passfail": - pass - - added_row = True - - if not added_row or (self.no_extraSteps is False): - self.add_row() - self.del_rows() - - - - def clear_gui(self): - for row in getattr(self, "rows", []): - # remove row widgets from layout - for key in ["name_edit", "type_combo", "value_edit", "value_edit2", "file_button"]: - widget = row.get(key) - if widget: - widget.setParent(None) - widget.deleteLater() - # remove options table if exists - table = row.get("options_widget") - if table: - table.setParent(None) - table.deleteLater() - self.rows = [] diff --git a/tests/unit_tests/test_recipe_creator.py b/tests/unit_tests/test_recipe_creator.py deleted file mode 100644 index e41ff6e..0000000 --- a/tests/unit_tests/test_recipe_creator.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2025 CERN -# -# SPDX-License-Identifier: LGPL-2.1-or-later -# import pytest -# from unittest.mock import patch, mock_open, MagicMock -# from PySide6.QtWidgets import QTreeWidgetItem -# from pypts.recipe_creator import RecipeEditorMainMenu, HashableTreeItem -# -# -# @pytest.fixture -# def editor(qtbot): -# # Create app instance, qtbot is pytest-qt fixture for Qt testing -# w = RecipeEditorMainMenu() -# qtbot.addWidget(w) -# return w -# -# -# def test_mark_required_field_colors_text(editor): -# # Create a dummy QTreeWidgetItem -# item = QTreeWidgetItem(["field", "value"]) -# # Initially no foreground set -# editor.mark_required_field(item, True) -# # The foreground color of column 0 should be the orange star color (210, 40, 0) -# color = item.foreground(0).color() -# assert color.red() == 210 -# assert color.green() == 40 -# assert color.blue() == 0 -# -# -# def test_extract_item_data_simple_and_complex(editor): -# # Build tree manually -# root = QTreeWidgetItem(["root", ""]) -# # Child scalar leaf -# leaf1 = QTreeWidgetItem(root, ["leaf1", "value1"]) -# # Child with children, map structure -# parent = QTreeWidgetItem(root, ["parent", ""]) -# child1 = QTreeWidgetItem(parent, ["key1", "val1"]) -# child2 = QTreeWidgetItem(parent, ["key2", "val2"]) -# -# # Extract data from root -# result = editor.extract_item_data(root) -# assert isinstance(result, dict) -# assert "leaf1" in result and result["leaf1"] == "value1" -# assert "parent" in result -# assert result["parent"] == {"key1": "val1", "key2": "val2"} -# -# -# def test_extract_item_data_lists(editor): -# root = QTreeWidgetItem(["root", ""]) -# child0 = QTreeWidgetItem(root, ["[0]", "val0"]) -# child1 = QTreeWidgetItem(root, ["[1]", "val1"]) -# child2 = QTreeWidgetItem(root, ["[2]", "val2"]) -# -# result = editor.extract_item_data(root) -# assert isinstance(result, list) -# assert result == ["val0", "val1", "val2"] -# -# -# def test_on_tree_item_clicked_highlights_line(editor): -# # Setup item and line mapping -# item = QTreeWidgetItem(["key", "value"]) -# hash_item = HashableTreeItem(item) -# editor.item_to_line[hash_item] = 42 -# -# # Patch highlight_line method to track calls -# with patch.object(editor, "highlight_line") as mock_highlight: -# editor.on_tree_item_clicked(item, 0) -# mock_highlight.assert_called_once_with(42) -# -# -# def test_on_tree_item_clicked_no_line_no_highlight(editor): -# item = QTreeWidgetItem(["key", "value"]) -# with patch.object(editor, "highlight_line") as mock_highlight: -# editor.on_tree_item_clicked(item, 0) -# mock_highlight.assert_not_called() -# -# -# def test_log_appends_message(editor): -# prev_count = editor.log_console.document().blockCount() -# editor.log("Test log message") -# new_count = editor.log_console.document().blockCount() -# assert new_count == prev_count + 1 -# last_text = editor.log_console.toPlainText().splitlines()[-1] -# assert "Test log message" in last_text -# -# -# @patch("builtins.open", new_callable=mock_open, read_data="key: value\n") -# def test_load_yaml_success(mock_file, editor): -# # Patch QFileDialog to return a known file path -# with patch("PySide6.QtWidgets.QFileDialog.getOpenFileName", return_value=("dummy.yaml", "YAML Files (*.yaml)")): -# editor.load_yaml_path() -# -# # After loading, current_file_path should be set -# assert editor.current_file_path == "dummy.yaml" -# # Tree should have top level items (document root) -# assert editor.tree.topLevelItemCount() > 0 -# -# -# @patch("builtins.open", new_callable=mock_open, read_data="invalid: [yaml") -# def test_load_yaml_parse_error(mock_file, editor): -# # Patch QFileDialog to return a known file path -# with patch("PySide6.QtWidgets.QFileDialog.getOpenFileName", return_value=("bad.yaml", "YAML Files (*.yaml)")): -# editor.load_yaml_path() -# -# # Log console should contain a parse error message -# logs = editor.log_console.toPlainText() -# assert "YAML parse error" in logs -# -# -# def test_on_save_recipe_clicked_no_file(editor): -# editor.current_file_path = "" -# editor.log_console.clear() -# editor.on_save_recipe_clicked() -# logs = editor.log_console.toPlainText() -# assert "No YAML file loaded" in logs -# -# -# -# def test_toggle_dark_mode_changes_style_and_logs(editor): -# editor.log_console.clear() -# editor.toggle_dark_mode(True) -# assert "Dark Mode enabled" in editor.log_console.toPlainText() -# editor.toggle_dark_mode(False) -# assert "Light Mode restored" in editor.log_console.toPlainText() diff --git a/tests/unit_tests/test_yamview_schema.py b/tests/unit_tests/test_yamview_schema.py index ef5c0e2..fda94c0 100644 --- a/tests/unit_tests/test_yamview_schema.py +++ b/tests/unit_tests/test_yamview_schema.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: LGPL-2.1-or-later """Schema-driven YamVIEW form tests.""" +import pytest + from pypts.recipe_language import ( INPUT_MODELS, OUTPUT_MODELS, @@ -11,6 +13,7 @@ DiscriminatedMappingWidget, Step_setup, recipe_form_description, + schema_widget_kind, ) @@ -57,3 +60,140 @@ def test_step_dialog_round_trips_discriminated_mapping_rows(qapp, qtbot): assert authored["input_mapping"]["wait_time"]["type"] == "direct" assert authored["input_mapping"]["wait_time"]["value"] == 0 assert authored["output_mapping"]["verdict"]["type"] == "passfail" + + +STEP_SAMPLES = { + "PythonModuleStep": { + "action_type": "method", + "module": "example_tests.py", + "method_name": "run", + }, + "SequenceStep": {"sequence": {"type": "internal", "name": "Other"}}, + "UserInteractionStep": {}, + "WaitStep": {"input_mapping": {"wait_time": {"type": "direct", "value": 0}}}, + "UserLoadingStep": { + "file_save_location": {"type": "local", "variable": "selected"} + }, + "UserRunMethodStep": { + "trigger_response": {"run": True}, + "action_type": "method", + "module": "example_tests.py", + "method_name": "run", + }, + "UserWriteStep": {}, + "SerialNumberStep": {}, + "SSHConnectStep": {}, + "SSHCloseStep": {}, + "SSHUploadStep": { + "files": [{"local": "payload.bin", "remote": "/tmp/payload.bin"}], + "permissions": "0755", + }, +} + + +@pytest.mark.parametrize("step_type", STEP_SAMPLES) +def test_every_step_discriminator_round_trips_through_one_dialog(qtbot, step_type): + dialog = Step_setup() + qtbot.addWidget(dialog) + node = { + "steptype": step_type, + "step_name": f"Example {step_type}", + "description": f"A complete {step_type} example.", + "input_mapping": {}, + "output_mapping": {}, + **STEP_SAMPLES[step_type], + } + + dialog.load_definition(node) + dialog.accept() + + assert dialog.result_step["_node"]["steptype"] == step_type + assert dialog.result_step["_node"]["description"] == node["description"] + + +@pytest.mark.parametrize( + ("mapping_kind", "values"), + [ + ( + "inputs", + { + "literal": { + "type": "direct", + "value": [1, {"nested": True}], + "indexed": False, + }, + "local": {"type": "local", "local_name": "inside"}, + "global": {"type": "global", "global_name": "shared"}, + "method": {"type": "method", "value": {"call": "helper"}}, + }, + ), + ( + "outputs", + { + "pass": {"type": "passfail"}, + "equal": {"type": "equals", "value": {"answer": 42}}, + "range": {"type": "range", "min": 0, "max": 10}, + "nested": {"type": "passthrough"}, + "local": {"type": "local", "local_name": "inside"}, + "global": {"type": "global", "global_name": "shared"}, + "image": {"type": "image"}, + }, + ), + ], +) +def test_every_mapping_discriminator_round_trips(qtbot, mapping_kind, values): + widget = DiscriminatedMappingWidget(mapping_kind) + qtbot.addWidget(widget) + widget.load_values(values) + + assert widget.values() == values + + +def test_schema_widget_selection_covers_structured_and_primitive_json_values(): + assert schema_widget_kind({"type": "array"}) == "structured" + assert schema_widget_kind({"type": "object"}) == "structured" + assert schema_widget_kind({}) == "structured" + assert schema_widget_kind({"type": "boolean"}) == "boolean" + assert schema_widget_kind({"type": "number"}) == "number" + assert schema_widget_kind({"type": "string"}) == "text" + + +def test_invalid_structured_json_is_reported_without_committing(qtbot): + dialog = Step_setup() + qtbot.addWidget(dialog) + dialog._skip_warning = True + dialog.list_steptype.setCurrentText("SSHUploadStep") + files = dialog.schema_form.field_widgets["files"] + files.setPlainText("[not valid JSON") + + with pytest.raises(ValueError, match="files.*valid JSON"): + dialog.schema_form.values() + + assert not hasattr(dialog, "result_step") + + +def test_switching_step_type_retains_compatible_common_fields(qtbot): + dialog = Step_setup() + qtbot.addWidget(dialog) + dialog.load_definition( + { + "steptype": "UserInteractionStep", + "step_name": "Keep me", + "description": "Common description", + "skip": True, + "input_mapping": {"message": {"type": "direct", "value": "Hello"}}, + "output_mapping": {}, + } + ) + dialog._skip_warning = True + dialog.list_steptype.setCurrentText("UserWriteStep") + + values = dialog.schema_form.values() + assert values["step_name"] == "Keep me" + assert values["description"] == "Common description" + assert values["skip"] is True + assert values["input_mapping"]["message"] == { + "type": "direct", + "value": "Hello", + "indexed": False, + } diff --git a/tests/unit_tests/test_yamview_workflows.py b/tests/unit_tests/test_yamview_workflows.py new file mode 100644 index 0000000..2578f20 --- /dev/null +++ b/tests/unit_tests/test_yamview_workflows.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: 2026 CERN +# SPDX-License-Identifier: LGPL-2.1-or-later +"""End-to-end working-state tests for the YamVIEW editor shell.""" + +from pathlib import Path +from types import SimpleNamespace + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import QDialog + +from pypts.recipe_language import Sequence +from pypts.recipe_parser import parse_recipe_text, recipe_to_yaml +from pypts.YamVIEW.customGUIModules import RecipeCreatorApp +from pypts.YamVIEW.recipe_creator import RecipeEditorMainMenu +from pypts.YamVIEW.recipe_sequencer_setup import _sequence_node + +RECIPE_PATH = Path(__file__).parents[2] / "src" / "pypts" / "recipes" / "simple_recipe.yml" + + +def _window_with_recipe(qtbot): + window = RecipeEditorMainMenu() + qtbot.addWidget(window) + assert window.load_yaml_recipe(RECIPE_PATH) + return window + + +def test_open_valid_recipe_populates_sequencer_and_enables_save(qtbot): + window = _window_with_recipe(qtbot) + + assert window.is_recipe_valid + assert window.sequencer.isEnabled() + assert window.save_action.isEnabled() + assert window.save_as_action.isEnabled() + assert window.sequencer.steps[0]["steptype"] == "preamble" + assert window.sequencer.steps[1]["steptype"] == "sequence_folder" + + +def test_structured_edit_preserves_exact_discriminator_and_stage(qtbot): + window = _window_with_recipe(qtbot) + sequence = window.sequencer.steps[1] + main = sequence["children"][1] + step = main["children"][0] + step["_node"]["description"] = "Changed through the structured editor." + + window.on_sequencer_updated(window.sequencer.steps) + + reparsed = parse_recipe_text(window.temporary_recipe_contents) + assert reparsed.is_valid + authored = reparsed.require_recipe().sequences[0].steps[0] + assert authored.steptype == "UserInteractionStep" + assert authored.description == "Changed through the structured editor." + assert step["_parent"] == "main_folder" + + +def test_add_and_edit_step_actions_keep_selected_stage_identity(qtbot, monkeypatch): + window = _window_with_recipe(qtbot) + sequence = window.sequencer.steps[1] + teardown = sequence["children"][2] + window.sequencer.expanded = True + window.sequencer.refresh() + for index in range(window.sequencer.list_widget.count()): + item = window.sequencer.list_widget.item(index) + if item.data(Qt.UserRole).get("_id") == teardown["_id"]: + window.sequencer.list_widget.setCurrentItem(item) + break + + class FakeStepDialog: + def __init__(self, parent=None): + self.result_step = { + "steptype": "UserInteractionStep", + "step_name": "Added", + "_node": { + "steptype": "UserInteractionStep", + "step_name": "Added", + "description": "Added through the dialog action.", + "input_mapping": {}, + "output_mapping": {}, + }, + "_id": "added-step", + } + + def exec(self): + return QDialog.Accepted + + monkeypatch.setattr( + "pypts.YamVIEW.recipe_sequencer_setup.Step_setup", FakeStepDialog + ) + window.sequencer.on_add_step() + added = teardown["children"][-1] + assert added["_parent"] == "teardown_folder" + assert added["_sequence_id"] == sequence["_sequence_id"] + + replacement = { + **added, + "step_name": "Edited", + "_node": {**added["_node"], "step_name": "Edited"}, + } + window.sequencer.current_setup_window = SimpleNamespace(result_step=replacement) + window.sequencer._finish_edit(QDialog.Accepted, added) + + assert teardown["children"][-1]["step_name"] == "Edited" + assert teardown["children"][-1]["_parent"] == "teardown_folder" + + +def test_step_move_and_delete_use_stable_sequence_and_folder_identity(qtbot): + window = _window_with_recipe(qtbot) + sequence = window.sequencer.steps[1] + main = sequence["children"][1] + teardown = sequence["children"][2] + step = main["children"][0] + + assert window.sequencer.move_step( + step, sequence["_sequence_id"], "teardown_folder", 0 + ) + assert step in teardown["children"] + assert step not in main["children"] + assert step["_parent"] == "teardown_folder" + + assert window.sequencer.delete_node(step, confirm=False) + assert step not in teardown["children"] + + +def test_new_sequence_is_kept_in_document_order(qtbot): + window = _window_with_recipe(qtbot) + sequence = Sequence( + sequence_name="Other", + description="Second sequence.", + parameters={}, + outputs={}, + locals={}, + setup_steps=[], + steps=[], + teardown_steps=[], + ) + document = sequence.model_dump(mode="python", by_alias=True, exclude_none=True) + window.sequencer.steps.append(_sequence_node(document, "sequence:other")) + + window.on_sequencer_updated(window.sequencer.steps) + + parsed = parse_recipe_text(window.temporary_recipe_contents) + assert [item.sequence_name for item in parsed.require_recipe().sequences] == [ + "Main", + "Other", + ] + + +def test_structurally_invalid_sequence_deletion_is_rolled_back(qtbot): + window = _window_with_recipe(qtbot) + original_text = window.temporary_recipe_contents + only_sequence = window.sequencer.steps[1] + + assert not window.sequencer.delete_node(only_sequence, confirm=False) + + assert window.temporary_recipe_contents == original_text + assert window.is_recipe_valid + assert len(window.sequencer.steps) == 2 + + +def test_semantically_invalid_gui_edit_is_retained_and_blocks_save(qtbot): + window = _window_with_recipe(qtbot) + valid_text = window.last_valid_recipe + window.sequencer.steps[0]["_node"]["main_sequence"] = "Missing" + + window.on_sequencer_updated(window.sequencer.steps) + + assert "main_sequence: Missing" in window.temporary_recipe_contents + assert not window.is_recipe_valid + assert window.sequencer.isEnabled() + assert not window.save_action.isEnabled() + assert not window.save_as_action.isEnabled() + assert window.last_valid_recipe == valid_text + + window.on_action_restore_recipe_clicked() + assert window.is_recipe_valid + assert window.temporary_recipe_contents == valid_text + + +def test_schema_invalid_raw_edit_is_retained_and_disables_sequencer(qtbot): + window = _window_with_recipe(qtbot) + invalid = window.temporary_recipe_contents.replace( + "UserInteractionStep", "userinteractionstep", 1 + ) + + window.yaml_viewer.setText(invalid) + + assert window.temporary_recipe_contents == invalid + assert "userinteractionstep" in window.temporary_recipe_contents + assert not window.is_recipe_valid + assert not window.sequencer.isEnabled() + assert window.yaml_viewer.extraSelections() + + +def test_canonical_save_uses_recipe_to_yaml(tmp_path, qtbot): + window = _window_with_recipe(qtbot) + destination = tmp_path / "saved.yaml" + expected = recipe_to_yaml(parse_recipe_text(window.temporary_recipe_contents).require_recipe()) + + assert window._write_recipe(destination) + + assert destination.read_text(encoding="utf-8") == expected + assert destination.read_text(encoding="utf-8").startswith("---\n") + assert parse_recipe_text(destination.read_text(encoding="utf-8")).is_valid + + +def test_save_as_uses_selected_path_and_canonical_output(tmp_path, qtbot, monkeypatch): + window = _window_with_recipe(qtbot) + destination = tmp_path / "save-as.yaml" + monkeypatch.setattr( + "pypts.YamVIEW.recipe_creator.QFileDialog.getSaveFileName", + lambda *args, **kwargs: (str(destination), "YAML Files (*.yaml *.yml)"), + ) + + window.on_save_as_clicked() + + assert window.current_file_path == str(destination) + assert parse_recipe_text(destination.read_text(encoding="utf-8")).is_valid + assert destination.read_text(encoding="utf-8").startswith("---\n") + + +def test_new_recipe_wizard_template_is_valid_v2_and_deterministic(): + generator = RecipeCreatorApp() + data = { + "name": "Generated", + "version": "1.0", + "recipe_version": "2.0.0", + "description": "Generated through YamVIEW.", + "main_sequence": "Main", + "num_steps": 1, + } + + generated = generator.generate_template_yaml(data) + parsed = parse_recipe_text(generated) + + assert parsed.is_valid + assert recipe_to_yaml(parsed.require_recipe()) in generated From 002001ed1f9af829d1cb21232c0341ec98ad907f Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 12:56:51 +0200 Subject: [PATCH 12/14] further extend yamview testing, also fixed light theme checkbox visibility --- docs/source/gui_architecture.rst | 5 ++ src/pypts/YamVIEW/recipe_sequencer_setup.py | 36 +++++----- src/pypts/YamVIEW/recipe_step_setup.py | 12 +++- src/pypts/gui_components/styles.py | 19 +++++ tests/unit_tests/test_gui_theme.py | 15 +++- tests/unit_tests/test_yamview_schema.py | 77 +++++++++++++++++++++ tests/unit_tests/test_yamview_workflows.py | 5 ++ 7 files changed, 145 insertions(+), 24 deletions(-) diff --git a/docs/source/gui_architecture.rst b/docs/source/gui_architecture.rst index e25067b..c607915 100644 --- a/docs/source/gui_architecture.rst +++ b/docs/source/gui_architecture.rst @@ -158,6 +158,11 @@ responsibilities: types, defaults, and field descriptions. * ``ScintillaYamlEditor`` owns editable source text and diagnostic highlighting. +The sequencer displays executable sequence documents only. The recipe-header +document (historically labelled ``Preamble``) remains part of the internal +aggregate and is editable in the YAML pane, but is not shown as an inactive +sequencer row. + Editor data flow ~~~~~~~~~~~~~~~~ diff --git a/src/pypts/YamVIEW/recipe_sequencer_setup.py b/src/pypts/YamVIEW/recipe_sequencer_setup.py index bf36711..d630381 100644 --- a/src/pypts/YamVIEW/recipe_sequencer_setup.py +++ b/src/pypts/YamVIEW/recipe_sequencer_setup.py @@ -146,6 +146,8 @@ def refresh(self) -> None: self.list_widget.clear() def add_node(node: dict[str, Any], indent: int = 0) -> None: + if node.get("steptype") == "preamble": + return is_folder = node.get("steptype") in FOLDER_TYPES | {"sequence_folder"} if is_folder: key = self._node_key(node) @@ -173,26 +175,20 @@ def add_node(node: dict[str, Any], indent: int = 0) -> None: descendant.setHidden(True) return - if node.get("steptype") == "preamble": - item = QListWidgetItem() - item.setFlags(Qt.ItemIsEnabled | Qt.ItemIsSelectable) - item.setData(Qt.UserRole, node) - widget = _build_header_widget(node["step_name"], indent) - else: - item = QListWidgetItem() - item.setFlags( - Qt.ItemIsEnabled - | Qt.ItemIsSelectable - | Qt.ItemIsDragEnabled - | Qt.ItemIsDropEnabled - ) - item.setData(Qt.UserRole, node) - block = StepBlock(node.get("step_name", "Unnamed step"), node) - widget = QFrame() - row = QHBoxLayout(widget) - row.setContentsMargins(indent + 4, 2, 4, 2) - row.addWidget(block) - widget.setMinimumHeight(block.minimumHeight() + 4) + item = QListWidgetItem() + item.setFlags( + Qt.ItemIsEnabled + | Qt.ItemIsSelectable + | Qt.ItemIsDragEnabled + | Qt.ItemIsDropEnabled + ) + item.setData(Qt.UserRole, node) + block = StepBlock(node.get("step_name", "Unnamed step"), node) + widget = QFrame() + row = QHBoxLayout(widget) + row.setContentsMargins(indent + 4, 2, 4, 2) + row.addWidget(block) + widget.setMinimumHeight(block.minimumHeight() + 4) item.setSizeHint(_item_size_for(widget)) self.list_widget.addItem(item) self.list_widget.setItemWidget(item, widget) diff --git a/src/pypts/YamVIEW/recipe_step_setup.py b/src/pypts/YamVIEW/recipe_step_setup.py index 4437861..447bac4 100644 --- a/src/pypts/YamVIEW/recipe_step_setup.py +++ b/src/pypts/YamVIEW/recipe_step_setup.py @@ -87,11 +87,13 @@ def recipe_form_description( def variants(name: str) -> dict[str, Any]: result = {} + discriminator_field = schema["$defs"][name]["discriminator"]["propertyName"] for discriminator, definition in discriminator_schemas(name, schema).items(): required = set(definition.get("required", [])) result[discriminator] = { "title": definition.get("title", discriminator), "description": definition.get("description", ""), + "discriminator_field": discriminator_field, "fields": { field_name: { **field_schema, @@ -156,11 +158,17 @@ def __init__(self, variant: dict[str, Any], parent=None): if description: layout.addWidget(QLabel(description)) for name, field in variant.get("fields", {}).items(): + if name == variant.get("discriminator_field"): + continue suffix = " *" if field.get("required") else "" - label = QLabel(f"{name}{suffix}") - label.setToolTip(field.get("description", "")) widget = build_schema_widget(field, self) self.field_widgets[name] = widget + if isinstance(widget, QCheckBox): + widget.setText(f"{name.replace('_', ' ').capitalize()}{suffix}") + layout.addWidget(widget) + continue + label = QLabel(f"{name}{suffix}") + label.setToolTip(field.get("description", "")) layout.addWidget(label) layout.addWidget(widget) diff --git a/src/pypts/gui_components/styles.py b/src/pypts/gui_components/styles.py index 90f2b7e..6d579a2 100644 --- a/src/pypts/gui_components/styles.py +++ b/src/pypts/gui_components/styles.py @@ -143,6 +143,25 @@ QPushButton:hover {{ background-color: #c8d8f4; }} +QCheckBox {{ + color: #1a1a2e; + spacing: 8px; +}} +QCheckBox::indicator {{ + width: 14px; + height: 14px; + border: 2px solid #1a1a2e; + border-radius: 3px; + background-color: #ffffff; +}} +QCheckBox::indicator:checked {{ + background-color: #1a1a2e; + border-color: #1a1a2e; +}} +QCheckBox::indicator:disabled {{ + border-color: #94a3b8; + background-color: #e2e8f0; +}} QPushButton#primaryBtn {{ background-color: {CERN_BLUE}; color: #ffffff; diff --git a/tests/unit_tests/test_gui_theme.py b/tests/unit_tests/test_gui_theme.py index 3e73bc3..b1f51d7 100644 --- a/tests/unit_tests/test_gui_theme.py +++ b/tests/unit_tests/test_gui_theme.py @@ -10,9 +10,9 @@ from PySide6.QtCore import Qt from pypts.gui import MainWindow -from pypts.YamVIEW.recipe_creator import RecipeEditorMainMenu -from pypts.gui_components.styles import CERN_BLUE +from pypts.gui_components.styles import CERN_BLUE, get_stylesheet from pypts.gui_theme import detect_system_dark_mode, install_system_theme_sync +from pypts.YamVIEW.recipe_creator import RecipeEditorMainMenu class _FakeSignal: @@ -106,6 +106,17 @@ def test_recipe_editor_uses_shared_light_palette(qtbot): window.close() +def test_light_theme_has_visible_checkbox_indicators_without_changing_dark_theme(): + light = get_stylesheet(False) + dark = get_stylesheet(True) + + assert "QCheckBox::indicator" in light + assert "border: 2px solid #1a1a2e" in light + assert "QCheckBox::indicator:checked" in light + assert "background-color: #1a1a2e" in light + assert "QCheckBox::indicator" not in dark + + def test_recipe_editor_toggle_dark_propagates_to_child_widgets(qtbot): window = RecipeEditorMainMenu() diff --git a/tests/unit_tests/test_yamview_schema.py b/tests/unit_tests/test_yamview_schema.py index fda94c0..8a7b89a 100644 --- a/tests/unit_tests/test_yamview_schema.py +++ b/tests/unit_tests/test_yamview_schema.py @@ -3,6 +3,7 @@ """Schema-driven YamVIEW form tests.""" import pytest +from PySide6.QtWidgets import QCheckBox, QMessageBox from pypts.recipe_language import ( INPUT_MODELS, @@ -62,6 +63,82 @@ def test_step_dialog_round_trips_discriminated_mapping_rows(qapp, qtbot): assert authored["output_mapping"]["verdict"]["type"] == "passfail" +def test_discriminator_has_one_authoritative_selector(qtbot): + dialog = Step_setup() + qtbot.addWidget(dialog) + + assert { + dialog.list_steptype.itemText(index) + for index in range(dialog.list_steptype.count()) + } == set(recipe_form_description()["steps"]) + assert "steptype" not in dialog.schema_form.field_widgets + + mapping = DiscriminatedMappingWidget("inputs") + qtbot.addWidget(mapping) + mapping.add_row("value", {"type": "direct", "value": 1}) + assert "type" not in mapping.rows[0]["form"].field_widgets + + +def test_boolean_fields_are_labeled_clickable_and_serialized(qtbot): + dialog = Step_setup() + qtbot.addWidget(dialog) + dialog.load_definition( + { + "steptype": "UserInteractionStep", + "step_name": "Boolean options", + "description": "Exercise all common execution flags.", + "skip": True, + "critical": False, + "continue_on_error": True, + "input_mapping": {}, + "output_mapping": {}, + } + ) + + expected = { + "skip": ("Skip", True), + "critical": ("Critical", False), + "continue_on_error": ("Continue on error", True), + } + for name, (label, checked) in expected.items(): + widget = dialog.schema_form.field_widgets[name] + assert isinstance(widget, QCheckBox) + assert widget.text() == label + assert widget.toolTip() + assert widget.isChecked() is checked + widget.click() + + dialog.accept() + authored = dialog.result_step["_node"] + assert authored["skip"] is False + assert authored["critical"] is True + assert authored["continue_on_error"] is False + + +def test_step_type_switch_uses_top_selector_and_honors_confirmation(qtbot, monkeypatch): + dialog = Step_setup() + qtbot.addWidget(dialog) + dialog.load_definition( + { + "steptype": "UserInteractionStep", + "step_name": "Switch me", + "description": "Keep common fields.", + "input_mapping": {}, + "output_mapping": {}, + } + ) + + monkeypatch.setattr(QMessageBox, "question", lambda *args, **kwargs: QMessageBox.Cancel) + dialog.list_steptype.setCurrentText("UserWriteStep") + assert dialog.list_steptype.currentText() == "UserInteractionStep" + + monkeypatch.setattr(QMessageBox, "question", lambda *args, **kwargs: QMessageBox.Ok) + dialog.list_steptype.setCurrentText("UserWriteStep") + assert dialog.list_steptype.currentText() == "UserWriteStep" + assert "steptype" not in dialog.schema_form.field_widgets + assert dialog.schema_form.values()["step_name"] == "Switch me" + + STEP_SAMPLES = { "PythonModuleStep": { "action_type": "method", diff --git a/tests/unit_tests/test_yamview_workflows.py b/tests/unit_tests/test_yamview_workflows.py index 2578f20..74956b6 100644 --- a/tests/unit_tests/test_yamview_workflows.py +++ b/tests/unit_tests/test_yamview_workflows.py @@ -33,6 +33,11 @@ def test_open_valid_recipe_populates_sequencer_and_enables_save(qtbot): assert window.save_as_action.isEnabled() assert window.sequencer.steps[0]["steptype"] == "preamble" assert window.sequencer.steps[1]["steptype"] == "sequence_folder" + visible_types = { + window.sequencer.list_widget.item(index).data(Qt.UserRole).get("steptype") + for index in range(window.sequencer.list_widget.count()) + } + assert "preamble" not in visible_types def test_structured_edit_preserves_exact_discriminator_and_stage(qtbot): From fa2978addf8fe44cc11a6d2fa3e241b87813b8ef Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 14:03:55 +0200 Subject: [PATCH 13/14] fix CI by adding the required .yml and example files to the package --- pyproject.toml | 14 ++++-- .../Minimal_setup/init_env_min.py | 29 ++++++------ .../Package_based_setup/init_env_pack.py | 46 +++++++++---------- tests/unit_tests/test_recipe_language.py | 12 +++-- 4 files changed, 57 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f814094..c3fa973 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,9 +78,17 @@ where = ["src"] include = ["pypts", "pypts.*"] [tool.setuptools.package-data] -# All package data which is tracked by git is included automatically due to -# include-package-data=true (the default) along with setuptools-scm. More details at: -# https://setuptools.pypa.io/en/latest/userguide/datafiles.html#include-package-data +"pypts.examples.environment_setup_tools.Minimal_setup" = [ + "Minimal_setup_recipe.yml", + "README.md", + "tests/example_tests.py", +] +"pypts.examples.environment_setup_tools.Package_based_setup" = [ + "Package_based_recipe.yml", + "README.md", + "pyproject.toml", + "tests/example_tests.py", +] [tool.setuptools_scm] # Tell setuptools_scm to write a _version.py file in the package. The diff --git a/src/pypts/examples/environment_setup_tools/Minimal_setup/init_env_min.py b/src/pypts/examples/environment_setup_tools/Minimal_setup/init_env_min.py index 3207095..56d557a 100644 --- a/src/pypts/examples/environment_setup_tools/Minimal_setup/init_env_min.py +++ b/src/pypts/examples/environment_setup_tools/Minimal_setup/init_env_min.py @@ -4,35 +4,38 @@ # Example test for initializing the minimal setup environment for a testing show of recipe and directory requirement. #It should initialize once called in a new directory, kept with a working directory. -from pypts.utils import get_project_root import shutil +from importlib.resources import as_file, files from pathlib import Path + def main(): project_root = Path.cwd() tests_dir = project_root / "tests" tests_dir.mkdir(exist_ok=True) - open(tests_dir/"__init__.py", "a").close() - #locating path of this script. will be used to copy the examples out. - package_root = Path(__file__).resolve().parent + (tests_dir / "__init__.py").touch(exist_ok=True) + package_resources = files(__package__) # Copy example recipe for minimal setup - recipe_src = package_root / "Minimal_setup_recipe.yml" recipe_dest = project_root / "Minimal_setup_recipe.yml" if not recipe_dest.exists(): - shutil.copy(recipe_src, recipe_dest) - print(f"Copied example recipe → {recipe_dest}") + with as_file(package_resources.joinpath("Minimal_setup_recipe.yml")) as recipe_src: + shutil.copy(recipe_src, recipe_dest) + print(f"Copied example recipe -> {recipe_dest}") else: print(f"Example recipe already exists at {recipe_dest}") # Copy Minimal setup tests - tests_src = package_root/ "tests" - for file in tests_src.glob("*.py"): - dest_file = tests_dir / file.name + tests_resources = package_resources.joinpath("tests") + for resource in tests_resources.iterdir(): + if not resource.name.endswith(".py"): + continue + dest_file = tests_dir / resource.name if not dest_file.exists(): - shutil.copy(file, dest_file) - print(f"Copied test file → {dest_file}") + with as_file(resource) as source: + shutil.copy(source, dest_file) + print(f"Copied test file -> {dest_file}") else: print(f"Test file already exists: {dest_file}") @@ -40,4 +43,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/pypts/examples/environment_setup_tools/Package_based_setup/init_env_pack.py b/src/pypts/examples/environment_setup_tools/Package_based_setup/init_env_pack.py index e26617e..38b5c53 100644 --- a/src/pypts/examples/environment_setup_tools/Package_based_setup/init_env_pack.py +++ b/src/pypts/examples/environment_setup_tools/Package_based_setup/init_env_pack.py @@ -5,11 +5,12 @@ #It should initialize once called in a new directory, kept with a working directory. -#Package-based architecture -from pypts.utils import get_project_root import shutil +import subprocess +import sys +from importlib.resources import as_file, files from pathlib import Path -import subprocess, sys + def main(): package_example = "example_package" @@ -25,39 +26,38 @@ def main(): resource_dir.mkdir(exist_ok=True) tests_dir.mkdir(exist_ok=True) - open(package_dir/"__init__.py", "a").close() - open(bin_dir/"__init__.py", "a").close() - open(resource_dir/"__init__.py", "a").close() - - - #locating path of this script. will be used to copy the examples out. - package_root = Path(__file__).resolve().parent + (package_dir / "__init__.py").touch(exist_ok=True) + (bin_dir / "__init__.py").touch(exist_ok=True) + (resource_dir / "__init__.py").touch(exist_ok=True) + package_resources = files(__package__) # Copy example recipe for minimal setup - recipe_src = package_root / "Package_based_recipe.yml" recipe_dest = resource_dir /"Package_based_recipe.yml" if not recipe_dest.exists(): - shutil.copy(recipe_src, recipe_dest) - print(f"Copied example recipe → {recipe_dest}") + with as_file(package_resources.joinpath("Package_based_recipe.yml")) as recipe_src: + shutil.copy(recipe_src, recipe_dest) + print(f"Copied example recipe -> {recipe_dest}") else: print(f"Example recipe already exists at {recipe_dest}") # Copy Minimal setup tests - tests_src = package_root/ "tests" - for file in tests_src.glob("*.py"): - dest_file = tests_dir / file.name + tests_resources = package_resources.joinpath("tests") + for resource in tests_resources.iterdir(): + if not resource.name.endswith(".py"): + continue + dest_file = tests_dir / resource.name if not dest_file.exists(): - shutil.copy(file, dest_file) - print(f"Copied test file → {dest_file}") + with as_file(resource) as source: + shutil.copy(source, dest_file) + print(f"Copied test file -> {dest_file}") else: print(f"Test file already exists: {dest_file}") - pyproject_toml = package_root / "pyproject.toml" - print(pyproject_toml) pyproject_dest = project_root/ "pyproject.toml" if not pyproject_dest.exists(): - shutil.copy(pyproject_toml, pyproject_dest) - print(f"Copied example recipe → {pyproject_dest}") + with as_file(package_resources.joinpath("pyproject.toml")) as pyproject_toml: + shutil.copy(pyproject_toml, pyproject_dest) + print(f"Copied example recipe -> {pyproject_dest}") else: print(f"Example recipe already exists at {pyproject_dest}") @@ -88,4 +88,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/unit_tests/test_recipe_language.py b/tests/unit_tests/test_recipe_language.py index 1cc5dfc..b2e3ab2 100644 --- a/tests/unit_tests/test_recipe_language.py +++ b/tests/unit_tests/test_recipe_language.py @@ -14,9 +14,8 @@ from pypts import recipe_language, recipe_parser from pypts.recipe_artifacts import ( - DEFAULT_REFERENCE_PATH, - DEFAULT_SCHEMA_PATH, render_json_schema, + write_artifacts, ) from pypts.recipe_language import ( INPUT_MODELS, @@ -407,12 +406,15 @@ def test_every_maintained_recipe_is_semantically_stable_when_serialized(path): assert second.require_recipe() == first -def test_generated_schema_and_reference_are_complete_and_current(): +def test_generated_schema_and_reference_are_complete_and_current(tmp_path): schema_text = render_json_schema() schema = json.loads(schema_text) reference = render_reference(schema) - assert schema_text == DEFAULT_SCHEMA_PATH.read_text(encoding="utf-8") - assert reference == DEFAULT_REFERENCE_PATH.read_text(encoding="utf-8") + schema_path = tmp_path / "recipe_language.schema.json" + reference_path = tmp_path / "recipe_language_reference.rst" + write_artifacts(schema_path, reference_path) + assert schema_text == schema_path.read_text(encoding="utf-8") + assert reference == reference_path.read_text(encoding="utf-8") definitions = schema["$defs"] for model in STEP_DEFINITION_MODELS + INPUT_MODELS + OUTPUT_MODELS: assert model.__name__ in definitions From a045cb143141d903e9d3e967e11be0d3917f78be Mon Sep 17 00:00:00 2001 From: alvaro Date: Fri, 14 Aug 2026 14:07:53 +0200 Subject: [PATCH 14/14] fix docs removing _static build --- docs/source/conf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cded14f..450afbd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,10 +48,6 @@ # html_theme = "alabaster" -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] html_show_sphinx = False html_show_sourcelink = True