From 19a69c693c70f51f938ffbafed8bcda9436df702 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:55:12 +0000 Subject: [PATCH] Merge the duplicate decision_log keys in the bridge YAML The database, sqlitedb and binaryTable entries each had two decision_log keys. YAML keeps the last, so the first block of each -- the notes on porting DID-matlab#229, #233 and #235, among others -- was silently dropped from the parsed entry. Each entry now has one decision_log holding both blocks, older notes first; no note text is changed. test_bridge_contract gains TestNoEntryWritesAKeyTwice, which loads every bridge file with a loader that reports a key written twice in one mapping. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012MroTijyNKsCMqFHpXFbpc --- src/did/did_matlab_python_bridge.yaml | 83 ++++++++++--------- src/did/did_matlab_python_bridge_file.yaml | 11 ++- ..._matlab_python_bridge_implementations.yaml | 17 ++-- tests/test_bridge_contract.py | 62 ++++++++++++++ 4 files changed, 118 insertions(+), 55 deletions(-) diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 2f6c6a9..580c623 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -47,6 +47,36 @@ classes: inherits: "abc.ABC" out_of_sync: false decision_log: > + Python Database class closely mirrors MATLAB did.database. + Schema validation (validate_docs, get_document_schema, + validate_doc_vs_schema, validate_field_type_and_value, checkfiles and + friends) was ported 2026-08-28 and brought the entry up to MATLAB + 0142532, PR #153 included. exist_doc was ported 2026-08-30 + (DID-python#42), onto SQLiteDB as open_doc is; a few MATLAB methods + (freeze_branch, is_branch_editable, display_branches, close_doc) are + still not ported. Deviation: MATLAB's add_docs also disables SQLite + journalling when validation is off; Python does not, since its sqlite3 + connection is managed differently (see the sqlitedb open_db/close_db + note). Synchronized 2026-03-15; validation 2026-08-28. + + DID-matlab#227 (bb39399, 2026-10-24) changed add_docs(..., 'Validate', false) + to set journal_mode=MEMORY instead of OFF, since each document is now + one transaction and ROLLBACK is undefined with the journal OFF. + Nothing to port: Python's add_docs has no such fast path and never + changes journal_mode. The rest of that commit is in sqlitedb.m; see + the sqlitedb entry. + + DID-matlab a7ed185 (DID-matlab#235, 2026-10-24): query_struct_to_sql_str + matches depends_on with an anchored, case-sensitive instr on + meta.depends_on (';NAME,VALUE;', or ',VALUE;' for '*') instead of LIKE + "%NAME,VALUE;%", and search_doc_ids brute-forces (fieldsearch) a + depends_on whose name or value is empty or holds ',' / ';'. That is + what SQLiteDB's _depends_on_sql_clause already did (DID-python #101), + so nothing is ported; tests/test_depends_on_parity.py mirrors + DID-matlab's TestDependsOnParity case for case. + DID-matlab 124cf5e (same PR) only fixes how MATLAB's brute_force_search + builds its did.query; nothing to port. + DID-matlab#229 (dae960b..83a2bda, 2026-10-24) made database.m's search fast. search_doc_ids searches the terms of an AND in rank order (base.id lookups, other terms, base.session_id, then the terms that test every @@ -57,11 +87,13 @@ classes: In Python the SQL search is SQLiteDB's, not Database's, so the port is in did/implementations/sqlitedb.py -- see the sqlitedb entry. Nothing changes in did/database.py. + DID-matlab 0de9cf2 (2026-10-24): search_doc_ids unions over did.datastructures.orOperands for an OR that is not all base.id lookups, instead of recursing once per nested OR. Ported to SQLiteDB._search_doc_ids (DID-python #100 / #103); nothing changes in did/database.py. + DID-matlab ec95ef7 (2026-10-24, DID-matlab#233 / DID-python#104): search_doc_ids hands an OR's operands to the new search_or_operands, which searches those that are single SQL terms up to orBatchTerms @@ -75,12 +107,23 @@ classes: term's complement in Python rather than in SQL (MATLAB batches them); (2) a batch whose SQL fails is retried one operand at a time so that each falls back as it would on its own. + DID-matlab 4f4e9aa..e4e02dc merges DID-matlab#235 into that branch: MATLAB now brute-forces the depends_on queries SQL cannot match exactly (isBruteForceDependsOn) and keeps them out of its OR batches. Python already did both -- _query_struct_to_sql_str returns None for them, so _search_or_operands leaves them to _search_doc_ids -- so nothing more is ported. + + DID-matlab 2c9d8fa7 (2026-10-24, DID-matlab#237): search_doc_ids used to + strip the '~' of a '~or' and search it as a plain 'or'; a negated OR now + returns the documents of the branch (or the candidates, inside an AND) + minus the OR's matches. Python already did exactly that -- + SQLiteDB._search_doc_ids takes universe - matched for a negated OR, and + _base_id_lookup_values treats a '~or' as no lookup -- so neither + did/database.py nor did/implementations/sqlitedb.py changes. + tests/test_negated_or.py pins the behaviour with the same cases as + MATLAB's did.unittest.TestNegatedOr. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had @@ -955,46 +998,6 @@ classes: MATLAB Hidden alias for display_branches, which is itself not ported. Added to the bridge 2026-08-28. - decision_log: > - Python Database class closely mirrors MATLAB did.database. - Schema validation (validate_docs, get_document_schema, - validate_doc_vs_schema, validate_field_type_and_value, checkfiles and - friends) was ported 2026-08-28 and brought the entry up to MATLAB - 0142532, PR #153 included. exist_doc was ported 2026-08-30 - (DID-python#42), onto SQLiteDB as open_doc is; a few MATLAB methods - (freeze_branch, is_branch_editable, display_branches, close_doc) are - still not ported. Deviation: MATLAB's add_docs also disables SQLite - journalling when validation is off; Python does not, since its sqlite3 - connection is managed differently (see the sqlitedb open_db/close_db - note). Synchronized 2026-03-15; validation 2026-08-28. - - DID-matlab#227 (bb39399, 2026-10-24) changed add_docs(..., 'Validate', false) - to set journal_mode=MEMORY instead of OFF, since each document is now - one transaction and ROLLBACK is undefined with the journal OFF. - Nothing to port: Python's add_docs has no such fast path and never - changes journal_mode. The rest of that commit is in sqlitedb.m; see - the sqlitedb entry. - - DID-matlab a7ed185 (DID-matlab#235, 2026-10-24): query_struct_to_sql_str - matches depends_on with an anchored, case-sensitive instr on - meta.depends_on (';NAME,VALUE;', or ',VALUE;' for '*') instead of LIKE - "%NAME,VALUE;%", and search_doc_ids brute-forces (fieldsearch) a - depends_on whose name or value is empty or holds ',' / ';'. That is - what SQLiteDB's _depends_on_sql_clause already did (DID-python #101), - so nothing is ported; tests/test_depends_on_parity.py mirrors - DID-matlab's TestDependsOnParity case for case. - DID-matlab 124cf5e (same PR) only fixes how MATLAB's brute_force_search - builds its did.query; nothing to port. - - DID-matlab 2c9d8fa7 (2026-10-24, DID-matlab#237): search_doc_ids used to - strip the '~' of a '~or' and search it as a plain 'or'; a negated OR now - returns the documents of the branch (or the candidates, inside an AND) - minus the OR's matches. Python already did exactly that -- - SQLiteDB._search_doc_ids takes universe - matched for a negated OR, and - _base_id_lookup_values treats a '~or' as no lookup -- so neither - did/database.py nor did/implementations/sqlitedb.py changes. - tests/test_negated_or.py pins the behaviour with the same cases as - MATLAB's did.unittest.TestNegatedOr. # ----------------------------------------------------------------------- # did.document diff --git a/src/did/did_matlab_python_bridge_file.yaml b/src/did/did_matlab_python_bridge_file.yaml index 08be096..c277207 100644 --- a/src/did/did_matlab_python_bridge_file.yaml +++ b/src/did/did_matlab_python_bridge_file.yaml @@ -307,6 +307,11 @@ classes: inherits_matlab: "handle" out_of_sync: false decision_log: > + SYNCHRONIZED 2026-08-29. Previously a partial port: read_row returned + raw bytes and insertRow, deleteRow, writeEntry, writeTable, findRow and + compare were absent. The full typed path is now present in both + languages over one shared little-endian on-disk layout. + SYNCHRONIZED 2026-09-08 (DID-python#66). resetLockState is now ported (see the method entry below) and the fileCache error paths that MATLAB commit 4a0f9a5 hardened are matched on the Python side: @@ -518,12 +523,6 @@ classes: 2026-10-23: the unsorted path reads the column once instead of reopening the file per row. Same result; the same change as DID-matlab#221. - decision_log: > - SYNCHRONIZED 2026-08-29. Previously a partial port: read_row returned - raw bytes and insertRow, deleteRow, writeEntry, writeTable, findRow and - compare were absent. The full typed path is now present in both - languages over one shared little-endian on-disk layout. - # ----------------------------------------------------------------------- # did.file.fileCache # ----------------------------------------------------------------------- diff --git a/src/did/did_matlab_python_bridge_implementations.yaml b/src/did/did_matlab_python_bridge_implementations.yaml index b8f8a59..111206a 100644 --- a/src/did/did_matlab_python_bridge_implementations.yaml +++ b/src/did/did_matlab_python_bridge_implementations.yaml @@ -37,6 +37,14 @@ classes: inherits_python: "Database" out_of_sync: false decision_log: > + Core database operations are synchronized. MATLAB recently changed + URL download mechanism (websave -> ndi.cloud.api.files.getFile) which + is MATLAB-ecosystem-specific; Python has no URL download path to + mirror. The 2026-06-12 SQL-injection hardening (14439fb) was authored + in lockstep with the Python parameterization/_sql_escape work already + present here. Last full sync: 2026-03-15; drift re-checked 2026-08-28 + against MATLAB 14439fb. + DID-matlab#229 (dae960b..83a2bda, 2026-10-24): fast id and isa searches, ported to SQLiteDB (_search_doc_ids, _search_query_head, _search_base_ids, _query_struct_to_sql_str, _ensure_indexes). MATLAB @@ -826,15 +834,6 @@ classes: different drives; MATLAB uses a prefix match with a trailing separator. - decision_log: > - Core database operations are synchronized. MATLAB recently changed - URL download mechanism (websave -> ndi.cloud.api.files.getFile) which - is MATLAB-ecosystem-specific; Python has no URL download path to - mirror. The 2026-06-12 SQL-injection hardening (14439fb) was authored - in lockstep with the Python parameterization/_sql_escape work already - present here. Last full sync: 2026-03-15; drift re-checked 2026-08-28 - against MATLAB 14439fb. - # ----------------------------------------------------------------------- # did.implementations.doc2sql (function module) # ----------------------------------------------------------------------- diff --git a/tests/test_bridge_contract.py b/tests/test_bridge_contract.py index fe22304..0f0bbd6 100644 --- a/tests/test_bridge_contract.py +++ b/tests/test_bridge_contract.py @@ -209,6 +209,68 @@ def test_different_names_sharing_one_matlab_file_are_not_flagged(self): ) +def duplicated_keys(text: str) -> list[str]: + """Mapping keys written twice in one mapping of a YAML document. + + Returns a description per offending mapping (line and keys); empty when + clean. + """ + found: list[str] = [] + + class _Loader(yaml.SafeLoader): + pass + + def construct_mapping(loader, node, deep=False): + keys = [loader.construct_object(k, deep=deep) for k, _ in node.value] + twice = sorted({str(k) for k in keys if keys.count(k) > 1}) + if twice: + found.append(f"line {node.start_mark.line + 1}: {', '.join(twice)}") + return yaml.SafeLoader.construct_mapping(loader, node, deep=deep) + + _Loader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping + ) + yaml.load(text, Loader=_Loader) # a SafeLoader subclass + return found + + +class TestNoEntryWritesAKeyTwice: + """A bridge entry gets ONE of each key. + + YAML keeps the last of two equal keys and drops the first without a word, + so a second `decision_log:` added to an entry hid every note in the first. + The database, sqlitedb and binaryTable entries each had one, and the notes + they hid were the record of DID-matlab#229, #233 and #235 being ported -- + present in the file, in review, and in no parsed entry. Nothing else + notices: every check reads the parsed YAML, which has no trace of the lost + block. + """ + + def test_no_bridge_file_repeats_a_key(self): + offenders = [ + f"{source.name} {where}" + for source in BRIDGE_FILES + for where in duplicated_keys(source.read_text(encoding="utf-8")) + ] + assert not offenders, ( + "these mappings write a key twice, so the first is silently " + "dropped:\n " + "\n ".join(offenders) + "\n\nMerge the two " + "values into one key." + ) + + def test_the_guard_would_catch_one(self): + text = ( + "classes:\n" + " - name: alpha\n" + " decision_log: first\n" + " status: ported\n" + " decision_log: second\n" + ) + offenders = duplicated_keys(text) + assert len(offenders) == 1 and "decision_log" in offenders[0] + assert duplicated_keys("classes:\n - name: a\n - name: b\n") == [] + + # --------------------------------------------------------------------------- # The status vocabulary, and the documentation of it # ---------------------------------------------------------------------------