diff --git a/.github/workflows/test-migrators-quick.yml b/.github/workflows/test-migrators-quick.yml index 2dbe55e0..88aa79ae 100644 --- a/.github/workflows/test-migrators-quick.yml +++ b/.github/workflows/test-migrators-quick.yml @@ -310,6 +310,18 @@ jobs: # EXISTS cannot notice something that stopped existing. run: python3 tools/test_batch_pass_wiring.py + - name: The batch-pass order is derived from declarations and checked (python, <1s) + # [migration reliability practices] (c) (Jess Haley, 2026-10-26, did-schema + # docs/plans/did_v2_go_forward_class_audit.md): each whole-dataset pass + # declares what it reads, adds, changes and removes; the order is + # derived from those declarations and every composition (the two + # report-writing call sites here; NDI-matlab's second pass in + # NDI-matlab's own CI, with --ndi) fails if a pass reads what a later + # pass writes, or reads what an earlier pass removed. + run: | + python3 tools/batch_pass_order.py + python3 tools/test_batch_pass_order.py + - name: Every migrator is in Contents.m and has a test (python, <1s) # +migrators_j/Contents.m is consulted as FACT about which classes # deliberately have NO migrator -- DID-schema/CLAUDE.md cites its diff --git a/src/did/+did2/+convert/epochMint.m b/src/did/+did2/+convert/epochMint.m index b39eedd7..7aefa6e2 100644 --- a/src/did/+did2/+convert/epochMint.m +++ b/src/did/+did2/+convert/epochMint.m @@ -51,6 +51,25 @@ % THERE, in +migrators_j, and credited to that migrator. Declaring them % here would count one emission twice. % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: epochid, entity(session), entity(epoch), +% acquisition_epoch, daqmetadatareader_epochdata_ingested, +% stimulus_response_scalar, syncrule_mapping, pyraview, +% epochfiles_ingested, method(method_parameters), acquisition_system, +% entity(element), directed_relation(element lineage), +% result.deferred_bath_anchors +% BATCH-PASS-ADDS: entity(epoch), relative_time_reference(epoch anchor), +% relative_reference(clock alignment endpoint), +% calculation(harmonic_component), +% entity(software minted by a batch pass), observation(pyraview), +% sampled_body(pyraview level), method(pyraview), clock_alignment, +% result.epoch_mint +% BATCH-PASS-CHANGES: relative_time_reference(bath anchor), +% daqmetadatareader_epochdata_ingested(epoch_id edge stamped) +% BATCH-PASS-REMOVES: stimulus_response_scalar, syncrule_mapping, pyraview +% % `acquisition_epoch` ADDED 2026-08-17, and it was a REAL OMISSION, not a % tidy-up. The #60 OPTION A loop below reads `acquisition_epoch` bodies (see % `if ~strcmp(rows(k).class_name, 'acquisition_epoch')`) and mints one diff --git a/src/did/+did2/+convert/foldGenericFiles.m b/src/did/+did2/+convert/foldGenericFiles.m index 79e8555f..437932c0 100644 --- a/src/did/+did2/+convert/foldGenericFiles.m +++ b/src/did/+did2/+convert/foldGenericFiles.m @@ -48,6 +48,20 @@ % where it is. % --------------------------------------------------------------------- % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: generic_file, ontologyLabel, ontology_label, +% fileReference, file_reference, observation(file carrier), +% sampled_body(lightsheet pyramid level), result.source_outputs +% BATCH-PASS-ADDS: observation(generic file), +% opaque_body(directory store by location), +% opaque_body(generic file content), +% relative_time_reference(generic file) +% BATCH-PASS-CHANGES: observation(file carrier), +% sampled_body(lightsheet pyramid level) +% BATCH-PASS-REMOVES: generic_file, fileReference, file_reference +% % --------------------------------------------------------------------- % STATUS: WRITTEN 2026-08-11 in a container with NO MATLAB AND NO OCTAVE % (`command -v matlab octave octave-cli` -> nothing, exit 1), so nothing here diff --git a/src/did/+did2/+convert/resolveClockAlignment.m b/src/did/+did2/+convert/resolveClockAlignment.m index e0450e23..91540731 100644 --- a/src/did/+did2/+convert/resolveClockAlignment.m +++ b/src/did/+did2/+convert/resolveClockAlignment.m @@ -21,6 +21,15 @@ % clock_alignment_configuration (and its software entity) and is % credited there; crediting it here would count that emission twice. % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: session, syncgraph, acquisition_system, syncrule +% BATCH-PASS-ADDS: clock_alignment_policy, clock_alignment_configuration, +% entity(software minted by a batch pass) +% BATCH-PASS-CHANGES: NONE -- each fold replaces its source (REMOVES, ADDS) +% BATCH-PASS-REMOVES: syncgraph, syncrule +% % `syncrule`: the DEVICE-PAIR rules are consumed here -- `filefind`, and the % channel-based `commonTriggersOverlappingEpochs` and `randomPulses`. A % file-based `filematch` rule reaches its `clock_alignment_configuration` in diff --git a/src/did/+did2/+convert/resolveDatasetEntities.m b/src/did/+did2/+convert/resolveDatasetEntities.m index 60cac2db..b49d3af3 100644 --- a/src/did/+did2/+convert/resolveDatasetEntities.m +++ b/src/did/+did2/+convert/resolveDatasetEntities.m @@ -30,6 +30,17 @@ % jMembershipBodies already skips "the dataset's own root".) % --------------------------------------------------------------------- % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: entity(dataset), entity(session), +% directed_relation(included_in) +% BATCH-PASS-ADDS: NONE -- it only keeps, drops or fills in what is there +% BATCH-PASS-CHANGES: entity(dataset), entity(session), +% directed_relation(included_in) +% BATCH-PASS-REMOVES: NONE -- it prunes duplicates and dangling edges +% (CHANGES), it folds no class away +% % 1. DEDUPLICATE dataset entities. Every dataset-level source % (metadata_editor / dataset_remote / session_in_a_dataset / % dataset_session_info) mints a `dataset` entity keyed on the SAME dataset diff --git a/src/did/+did2/+convert/resolveDeferredBaths.m b/src/did/+did2/+convert/resolveDeferredBaths.m index 0037c278..c3a27a00 100644 --- a/src/did/+did2/+convert/resolveDeferredBaths.m +++ b/src/did/+did2/+convert/resolveDeferredBaths.m @@ -31,6 +31,19 @@ % BATCH-PASS-EMITS: stimulus_bath -> document: relative_time_reference, % manipulation+formulation, formulation, chemical, observation+term % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: stimulus_bath, element, entity(element), +% directed_relation(element lineage), acquisition_epoch, element_epoch, +% result.passthrough +% BATCH-PASS-ADDS: relative_time_reference(bath anchor), +% manipulation(bath), formulation, chemical, observation(bath), +% result.deferred_bath_anchors +% BATCH-PASS-CHANGES: NONE -- it only appends; the deferred stimulus_bath +% entries leave the quarantine, not the batch +% BATCH-PASS-REMOVES: NONE -- it folds no document that is in the batch +% % SCOPE OF THAT LINE: V_eta is the only path. The V_zeta branch (makeBath % emitting `bath` + `pharmacological_manipulation`) was deleted with % +migrators_e/+migrators_i (did-schema #88, OPEN_WORK #128): only diff --git a/src/did/+did2/+convert/resolveEpochProbemap.m b/src/did/+did2/+convert/resolveEpochProbemap.m index f277bf31..f900b324 100644 --- a/src/did/+did2/+convert/resolveEpochProbemap.m +++ b/src/did/+did2/+convert/resolveEpochProbemap.m @@ -45,6 +45,29 @@ % emission twice. % BATCH-PASS-EMITS: element -> document: observation+voltage, observation+current, observation+intensity, observation+acceleration, observation+temperature, observation+time, opaque_body, relative_time_reference % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: acquisition_epoch, epochfiles_ingested, +% daqmetadatareader_epochdata_ingested, sampled_body(pyraview level), +% entity(epoch), acquisition_system, directed_relation(element lineage), +% directed_relation(observes), entity(element), +% calculation(spike sorter), observation(session-scoped recording), +% observation(pyraview), +% relative_time_reference(session anchor of a recording observation), +% relative_reference(folded session anchor), +% relative_time_reference(epoch anchor), result.epoch_mint +% BATCH-PASS-ADDS: observation(recording per epoch), +% manipulation(stimulator per epoch), opaque_body(epoch file by location), +% opaque_body(spike train), opaque_body(ingested archive), +% relative_time_reference(epoch anchor), relative_reference(epoch anchor), +% ingestion_manifest, directed_relation(wiring) +% BATCH-PASS-CHANGES: sampled_body(pyraview level), calculation(spike sorter) +% BATCH-PASS-REMOVES: acquisition_epoch, epochfiles_ingested, +% daqmetadatareader_epochdata_ingested, +% observation(session-scoped recording), observation(pyraview), +% directed_relation(observes) +% % --------------------------------------------------------------------- % THE EPOCH'S FILES ARE THE OBSERVATIONS' DATA BODIES (2026-10-21) % --------------------------------------------------------------------- diff --git a/src/did/+did2/+convert/resolveOpenmindsCitations.m b/src/did/+did2/+convert/resolveOpenmindsCitations.m index 2b46a72c..d5b425ce 100644 --- a/src/did/+did2/+convert/resolveOpenmindsCitations.m +++ b/src/did/+did2/+convert/resolveOpenmindsCitations.m @@ -22,6 +22,18 @@ % BATCH-PASS-EMITS: openminds -> document: dataset, person, organization, % funding, publication, web_resource, directed_relation % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% (dataset citation: the subject-side types -- Subject, Species, Strain, +% ... -- are left out of the walk, routeIsSubjectSide below.) +% BATCH-PASS-READS: openminds(dataset citation) +% BATCH-PASS-ADDS: entity(dataset), entity(person), entity(organization), +% funding, publication, web_resource, directed_relation(openMINDS role) +% BATCH-PASS-CHANGES: NONE -- it appends the assembled documents and +% removes the sources +% BATCH-PASS-REMOVES: openminds(dataset citation) +% % The seven are the six entity classes named in the H1 line plus the % `directed_relation` edges between them, all minted through % did2.convert.entities (entityDoc :586/:610/:664/:703/:731, orgFor diff --git a/src/did/+did2/+convert/resolveResponseParameters.m b/src/did/+did2/+convert/resolveResponseParameters.m index e64aa418..6edfe758 100644 --- a/src/did/+did2/+convert/resolveResponseParameters.m +++ b/src/did/+did2/+convert/resolveResponseParameters.m @@ -18,6 +18,15 @@ % BATCH-PASS-EMITS: stimulus_response_scalar_parameters_basic -> document: % method % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: stimulus_response_scalar_parameters_basic, +% calculation(harmonic_component) +% BATCH-PASS-ADDS: method(response parameters) +% BATCH-PASS-CHANGES: calculation(harmonic_component) +% BATCH-PASS-REMOVES: stimulus_response_scalar_parameters_basic +% % `inline` IS NOT A SOFTER `document`, AND THE DISTINCTION IS THE THIRD OF % THE THREE EMISSION SHAPES row 107 names. The destination is the % `subject_interaction.method_parameters` BLOCK, written at :328 onto the diff --git a/src/did/+did2/+convert/resolveSessionAnchors.m b/src/did/+did2/+convert/resolveSessionAnchors.m index 3f746adf..bad70daf 100644 --- a/src/did/+did2/+convert/resolveSessionAnchors.m +++ b/src/did/+did2/+convert/resolveSessionAnchors.m @@ -18,6 +18,16 @@ % BATCH-PASS-EMITS: session_relative_reference -> document: relative_reference % BATCH-PASS-EMITS: session_bounded_reference -> document: relative_reference % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: session, session_relative_reference, +% session_bounded_reference +% BATCH-PASS-ADDS: relative_reference(folded session anchor) +% BATCH-PASS-CHANGES: NONE -- each fold replaces its source under the same +% id (REMOVES, ADDS) +% BATCH-PASS-REMOVES: session_relative_reference, session_bounded_reference +% % NEITHER CONSUMED NAME IS A did_v1 SOURCE CLASS. Both are DID-side % intermediates -- V_zeta/V_eta time_reference classes that the per-document % migrators and resolveDeferredBaths mint on the way through -- so neither diff --git a/src/did/+did2/+convert/resolveValidIntervals.m b/src/did/+did2/+convert/resolveValidIntervals.m index ec16b680..82a25524 100644 --- a/src/did/+did2/+convert/resolveValidIntervals.m +++ b/src/did/+did2/+convert/resolveValidIntervals.m @@ -54,6 +54,17 @@ % decomposition behind the guard is NOT the signed model and waits on % `axes[]`. % +% WHAT IT READS AND WRITES (tools/batch_pass_order.py derives and checks the +% pass order from these; [migration reliability practices] (c), Jess Haley, +% 2026-10-26). Read from this file's code, class by class. +% BATCH-PASS-READS: valid_interval, entity(epoch), +% directed_relation(element lineage) +% BATCH-PASS-ADDS: observation(validity), +% relative_reference(validity interval) +% BATCH-PASS-CHANGES: NONE -- it appends; dormant by default (Decompose +% false), these are what its guarded branch writes +% BATCH-PASS-REMOVES: valid_interval +% % THAT `nothing` IS LOAD-BEARING AND MUST NOT BE "FIXED" INTO AN EMISSION. % The `valid_interval` ledger row carries `logical_observation` as its % DECIDED target, so declaring it emitted here would move that row to the diff --git a/tests/+did2/+unittest/testBatchPassWiring.m b/tests/+did2/+unittest/testBatchPassWiring.m index ea6f812a..026f92f7 100644 --- a/tests/+did2/+unittest/testBatchPassWiring.m +++ b/tests/+did2/+unittest/testBatchPassWiring.m @@ -470,6 +470,17 @@ 'have. Listed for the same reason bodyResolver is. COST: none of its ' ... 'own; the cost is stimulusPresentationToItem''s timing, stated in ' ... 'that row.']; ... +'v1FileReader', 'NDI', ... + ['Not a transform: a READER of a v1 document''s attached file that ' ... + 'ndi.migrate.local hands to did2.convert.v1_to_v2 as ''V1FileReader'' ' ... + '(did2.convert.v1Files), built on v1AttachedFile; a migrator that must ' ... + 'read an attached file (gene_list_mapping''s mapping.tsv) calls it. ' ... + 'It needs the session''s .ndi directory on disk, which the DID corpus ' ... + 'harness (document-only zips) does not have; there did2.convert.v1Files ' ... + 'has no reader, tries the file''s recorded location, and when that is ' ... + 'not on this computer the document is carried (NDI-matlab #1140). ' ... + 'COST: a DID corpus run carries every document whose migrator needs ' ... + 'an attached file that is not on the runner.']; ... 'controlRecordsByPresentation', 'NDI', ... ['Not a transform: a READER that groups the ORIGINAL v1 ' ... 'control_stimulus_ids bodies by the presentation each names, called ' ... diff --git a/tools/batch_pass_order.py b/tools/batch_pass_order.py new file mode 100644 index 00000000..3ee58322 --- /dev/null +++ b/tools/batch_pass_order.py @@ -0,0 +1,518 @@ +#!/usr/bin/env python3 +"""The order of the whole-dataset passes, DERIVED from what each declares it +reads and writes, and CHECKED against every place that composes them. + +THE DECISION. did-schema docs/plans/did_v2_go_forward_class_audit.md, Decided +by Jess Haley, 2026-10-26 [migration reliability practices], practice (c): +"whole-dataset passes (DID-matlab `resolve*` post-passes, NDI-matlab's second +pass) declare what they read and write, and their order is derived and +checked, not kept in comments." + +WHAT EACH PASS DECLARES. Four markers, in the same header comment block as +the CONSUMES / EMITS markers tools/batch_pass_declarations.py reads (those +credit the completion ladder; these say what the pass needs and changes, which +is a different question: a document a pass mints by re-running a migrator is +credited to that migrator, but it is still in the batch for the next pass). +Each value is a comma-separated list of ITEMS, or `NONE -- `: + + READS: what the pass selects from the batch and must see in full + ADDS: documents it appends to the batch + CHANGES: documents already in the batch that it edits, or prunes + some of (a duplicate dropped, an edge cut) + REMOVES: documents it folds away (a source it replaces) + +( is the BATCH-PASS- prefix; the tokens are assembled below rather +than spelled out, for the reason tools/batch_pass_declarations.py gives.) + +An ITEM is a class name (`entity`, `acquisition_epoch`), a class with a +qualifier in parentheses (`entity(dataset)`, `directed_relation(included_in)`) +or a value one pass hands the next outside the batch: a field of the batch +result (`result.deferred_bath_anchors`) or, in NDI-matlab, a variable of +ndi.migrate.local (`ndi.bath_anchors`). A qualifier narrows: `entity(dataset)` and +`entity(epoch)` are different documents, and an unqualified `entity` overlaps +every `entity(...)`. A qualifier is the declaring pass's statement of what it +selects; it is text, and it is matched exactly, so a qualifier that makes two +items disjoint is a claim the declaration's author makes and a reviewer +checks. + +What a pass reads only to COUNT (a report instrument that decides nothing) is +not a read here. A pass that folds the documents of a class it can handle and +leaves the rest for a later pass CHANGES that class; REMOVES is for documents +no later pass is meant to read. + +THE TWO RULES. For passes P and Q, P earlier in a composition than Q: + + R1 P READS an item Q ADDS or CHANGES -> P runs too early: it cannot see + what Q writes. ("A pass that consumes something emitted later.") + R2 Q READS an item P REMOVES -> Q runs too late: P has folded + away what Q reads. + +The derived order is the topological order of those constraints over the +members of a composition, ties broken by the composition's own order, so a +composition that breaks no rule derives to itself. A cycle means no order can +satisfy the declarations, and fails. + +WHAT IS COMPOSED, read from code, never from a list kept here: + * DID-matlab: each report-writing call site census_digest names + (runCorpusDiscovery, testCorpusPRED), its did2.convert.* calls in call + order, restricted to the derived chain (census_digest.harness_pass_chain); + * NDI-matlab (with --ndi ROOT, or $NDI_MATLAB): NDI's second pass in + +ndi/+migrate/local.m, step by step. A step is the code between two + `migrateStepMark('