From 3511e42e86b09dd365eb27c13850f2d16f0cc58d Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 14 Oct 2026 15:43:11 +0800 Subject: [PATCH 1/4] Separate open external JSON readers from closed owned contracts --- .github/workflows/ci.yml | 2 + .../JournalJSONBoundaryTests.swift | 90 +++++++++ .../2026-10-14-external-input-open-json.md | 68 +++++++ logseq_db_storage/lib/sync_outbox_store.ml | 8 +- logseq_db_worker/test/test_protocol.ml | 53 ++++- .../lib/effect_runner/protocol/catalog.ml | 6 +- .../lib/effect_runner/protocol/e2ee.ml | 11 +- logseq_sync/lib/pure_reducer/core.ml | 2 +- logseq_sync/lib/pure_reducer/sync_protocol.ml | 167 ++++++++++++--- logseq_sync/test/core_contract.ml | 58 ++++++ logseq_sync/test/sync_protocol_contract.ml | 190 +++++++++++++++--- logseq_sync/test/transport_contract.ml | 128 +++++++++++- swift/JournalCognitoNative.swift | 2 +- swift/JournalE2EECrypto.swift | 2 +- swift/JournalExtensions.swift | 2 +- swift/JournalInternalJSON.swift | 146 ++++++++++++++ swift/JournalList.swift | 2 +- swift/JournalLocalAccountBindingStore.swift | 3 +- swift/JournalNativeServices.swift | 2 +- swift/JournalPlatformWire.swift | 2 +- test/dune | 1 + test/source_boundary_test.ml | 3 +- test/test_wrapped_graph_key_crypto.sh | 1 + tool/build_authentication_probe.py | 2 +- tool/test_asset_interop.py | 4 +- tool/test_json_boundaries.py | 17 ++ tool/test_swiftui_cognito.py | 2 +- tool/test_swiftui_events.py | 4 +- tool/test_swiftui_platform.py | 2 +- tool/test_swiftui_services.py | 8 +- tool/test_swiftui_warm_start.py | 2 +- 31 files changed, 898 insertions(+), 92 deletions(-) create mode 100644 apple-tests/platform-wire/JournalJSONBoundaryTests.swift create mode 100644 docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md create mode 100644 swift/JournalInternalJSON.swift create mode 100644 tool/test_json_boundaries.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4846b9..778f7ac8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: - uses: actions/checkout@v4 - name: Verify Apple app icon packaging run: python3 tool/test_apple_app_icon.py + - name: Verify owned and external native JSON boundaries + run: python3 tool/test_json_boundaries.py - uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: "5.5.0" diff --git a/apple-tests/platform-wire/JournalJSONBoundaryTests.swift b/apple-tests/platform-wire/JournalJSONBoundaryTests.swift new file mode 100644 index 00000000..ff7bc8a6 --- /dev/null +++ b/apple-tests/platform-wire/JournalJSONBoundaryTests.swift @@ -0,0 +1,90 @@ +import Foundation + +@main @MainActor struct JournalJSONBoundaryTests { + private struct Child: Decodable { let name: String; let note: String? } + private struct Internal: Decodable { let children: [Child]; let named: [String: Child] } + private enum Choice: Decodable { case value(Child) } + private struct FoundationValues: Codable { + let data: Data; let date: Date; let url: URL; let decimal: Decimal + } + private static var failures = 0 + + private static func check(_ name: String, _ action: () throws -> Bool) { + do { + if try action() { print("PASS \(name)") } + else { failures += 1; print("FAIL \(name)") } + } catch { failures += 1; print("FAIL \(name): \(type(of: error))") } + } + private static func rejected(_ action: () throws -> Void) -> Bool { + do { try action(); return false } catch { return true } + } + private static func packet(_ payload: String) -> Data { + var bytes = Data([76,74,80,50,2,0,8,0] + Array(repeating: UInt8(0), count:24)) + let data = Data(payload.utf8) + for n in 0..<4 { bytes[24+n] = UInt8((data.count >> (8*n)) & 255) } + bytes.append(data) + return bytes + } + + static func main() throws { + check("public native wire rejects extra internal fields") { + rejected { _ = try JournalPlatformWire.decodeRequest(packet(#"{"challengeId":"synthetic","future":true}"#)) } + } + check("public native wire retains required and type validation") { + [#"{"future":true}"#, #"{"challengeId":null}"#, #"{"challengeId":1}"#].allSatisfy { + input in rejected { _ = try JournalPlatformWire.decodeRequest(packet(input)) } + } + } + check("public account binding rejects extra internal fields") { + rejected { _ = try JournalLocalAccountBindingStore.decode(Data(#"{"version":2,"userId":"synthetic","managedSyncOrigin":"https://example.invalid","future":true}"#.utf8)) } + } + check("recursive internal decoder rejects root nested array and dictionary extensions") { + [#"{"children":[],"named":{},"future":true}"#, + #"{"children":[{"name":"synthetic","future":true}],"named":{}}"#, + #"{"children":[],"named":{"dynamic-key":{"name":"synthetic","future":true}}}"#].allSatisfy { + input in rejected { _ = try JournalInternalJSON.decode(Internal.self, from: Data(input.utf8)) } + } + } + check("closed native decoder preserves option null and dynamic dictionaries") { + let value = try JournalInternalJSON.decode(Internal.self, from: + Data(#"{"children":[{"name":"synthetic","note":null}],"named":{"dynamic-key":{"name":"synthetic"}}}"#.utf8)) + return value.children.first?.note == nil && value.named["dynamic-key"]?.name == "synthetic" + } + check("closed native decoder preserves required and scalar type constraints") { + [#"{"children":[],"named":{"dynamic-key":{}}}"#, + #"{"children":[{"name":null}],"named":{}}"#, + #"{"children":[{"name":1}],"named":{}}"#].allSatisfy { + input in rejected { _ = try JournalInternalJSON.decode(Internal.self, from: Data(input.utf8)) } + } + } + check("closed enum containers reject extra cases") { + rejected { _ = try JournalInternalJSON.decode(Choice.self, from: + Data(#"{"value":{"_0":{"name":"synthetic"}},"future":{}}"#.utf8)) } + } + check("closed field errors do not emit unknown field names or values") { + do { + _ = try JournalInternalJSON.decode(Internal.self, from: + Data(#"{"children":[],"named":{},"CANARY_FIELD_NAME":"CANARY_PRIVATE_VALUE"}"#.utf8)) + return false + } catch { return !String(describing:error).contains("CANARY") } + } + check("external Cognito response keeps open semantics") { + let response = try JSONDecoder().decode(JournalCognitoTokenResponse.self, from: + Data(#"{"id_token":"synthetic","access_token":"synthetic","expires_in":3600,"token_type":"Bearer","future":{"nested":true}}"#.utf8)) + return response.expires_in == 3600 && response.refresh_token == nil + } + check("closed decoder preserves Foundation scalar roots and nested values") { + let encoder = JSONEncoder() + let bytes = Data([1, 2, 3]) + let root = try JournalInternalJSON.decode(Data.self, from: encoder.encode(bytes)) + let number = try JournalInternalJSON.decode(Decimal.self, from: encoder.encode(Decimal(12))) + let value = FoundationValues(data: bytes, date: Date(timeIntervalSinceReferenceDate: 1), + url: URL(string: "https://example.invalid")!, decimal: Decimal(12)) + let decoded = try JournalInternalJSON.decode(FoundationValues.self, from: encoder.encode(value)) + let array = try JournalInternalJSON.decode([Data].self, from: encoder.encode([bytes])) + return root == bytes && number == 12 && decoded.data == bytes && decoded.date == value.date + && decoded.url == value.url && decoded.decimal == value.decimal && array == [bytes] + } + if failures > 0 { exit(1) } + } +} diff --git a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md new file mode 100644 index 00000000..0f67635f --- /dev/null +++ b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md @@ -0,0 +1,68 @@ +# Open external JSON inputs and close owned JSON contracts + +## Problem + +A real Journal sync error rejects the extra tx-id field on pull/ok.txs[0] before any transaction is applied. The generated server DTO and manual structural whitelist both reject it. External services can extend responses without changing the fields Journal consumes. Shared input/output DTOs also obscure which side owns a schema; synthesized Swift internal decoders currently tolerate unknown fields. + +## Decision + +Follow the user-approved boundary policy. External server/HTTP JSON input DTOs derive open PPX readers, including nested records; validate declared required fields, their types and domain semantics. Continue rejecting unsupported message variants and malformed JSON. Ignore unused pull tx-id regardless of its value. Keep generated closed output DTOs separate, with semantic and wire budget checks; client wire validation remains closed. Internal OCaml commands, outcomes and persistence domain codecs remain closed. Swift-owned internal/native receivers use a recursive Decoder wrapper that rejects keys absent from synthesized CodingKeys; external Cognito and Transit readers do not use it. Keep generated Codable/PPX construction, existing keys and formats. + +The existing metadata-only outbox/format readers are projections of a complete record, not full-record admission or constructors; make decode-only derivations explicit where applicable. Their extra declared owner fields must not be rejected as projection extras. The domain owner remains responsible for the full closed contract. + +## Alternatives considered + +### Declare only tx-id + +Rejected because the user explicitly authorized open external inputs, including nested future fields. Adding a whitelist entry would leave the same failure for the next external extension. + +### Make all shared DTOs open + +Rejected because it would extend open semantics to internal receivers and weaken validation of owned output. Use separate external-input and closed-output DTOs. + +## Acceptance criteria + +- Public Sync_protocol API accepts top-level and nested server extensions while preserving required/type/null/semantic, cursor/budget and discriminator failures; all client input remains closed and generated outbound shapes contain only contract fields. +- Public Runner requests exercise open HTTP catalog/E2EE inputs through the existing isolated TLS peer. Core public completions retain snapshot metadata field/URL validation. +- Native public wire and closed-decoder utility tests reject root/nested/array extras, retain required/type/null/default behavior and leave external Cognito decode open. +- Tests are written before implementation and demonstrably RED for the old behavior, then GREEN. Existing targeted regressions and appropriate broader checks pass. +- No Dune/spec files, UI PR73 checkout, user graph, credentials, phone installation or temporary diagnostic observer is changed. +- Create a formal PR, freeze head SHA, wait for that exact head CI terminal results, then request independent journal-code-review. Do not merge. + +## Risks + +- Open schemas still enforce standard JSON safety and declared semantic constraints; unknown event types are not extension fields. +- Swift native receiver fields must match the complete OCaml producer contract. Nested checking must preserve optional nulls, arrays and dynamic dictionaries rather than guess field lists. +- Public Core receives already-decoded WS messages and HTTP key values, so it cannot reproduce raw field rejection. The narrowest raw WS boundary is public Sync_protocol; HTTP raw parsing belongs to the public Runner and its transport peer. Do not inject an already incorrect parsed result into a reducer as a reproduction. +- Existing live evidence covers the reported WS extra-field rejection and historical catalog/E2EE paths, plus a public JWKS read. Synthetic tests are not live service validation; actual signed-in response variants remain unobserved. + +## Questions + +- None. The user expressly authorized this policy change and a separate formal PR; any necessary Dune/spec edit would require a separate minimal-diff request. + +## Consequences + +WS server records now have separate input-only open PPX DTOs. The discriminator remains owned by its header reader and semantic validator. Open object validation checks declared values and duplicate safety without running the owned-field whitelist. Output uses the original closed DTO serializers and generated closed readers, plus existing semantic and size validation. HTTP E2EE inputs are open; catalog and snapshot metadata retain their already-open behavior but expose no encoders. Internal Config/Protocol/full persistence codecs stay closed. Decode-only outbox indexing and account-format projections remain explicitly separate from full-record admission. + +Swift native wire, extension properties, persisted token/binding/recovery records and the crypto bridge use the closed recursive Decoder. It derives allowed keys from each Decodable container's synthesized CodingKeys, including nested arrays/enums; dynamic dictionaries keep their intentional key space. Foundation's Data/Date/URL/Decimal decoding is preserved. External OAuth/JWT/JWKS/Transit readers keep their existing open behavior. Unknown-field diagnostics omit unknown names and values. + +The user explicitly approved the one-line JournalInternalJSON.swift dependency in the existing wrapped-key crypto Dune rule. It is the only Dune change; all OCaml spec files remain untouched. + +## Validation and actual-data coverage + +Tests were added first. With production at main953, the public WS codec failed its two new extension/declared-constraint tests, native wire/binding plus recursive utility behavior failed five extension assertions, and the public Runner/local TLS peer failed four E2EE extension-acceptance assertions. Existing required/type/null/duplicate controls passed. The Core metadata and internal Config controls passed at baseline. These fixtures are synthetic, including the independent local TLS peer and signed Cognito issuer. + +| Boundary | Real evidence | Regression coverage | +| --- | --- | --- | +| WS server/client | Materialized and viewed error screenshot proves pull/ok.txs[0].tx-id rejection; full payload and deployed SHA unknown | All supported server variants/rejection reasons, top/nested extensions, ignored tx-id shapes, required/type/null/semantic/variant errors, closed client/output and wire limits | +| Catalog and E2EE HTTP | Earlier authorized App type-only logs: catalog/user keys HTTP 200; graph key before/after PR72; no new signed-in response collection | Public Runner with independent localhost TLS peer, valid extensions and invalid consumed fields | +| Cognito OAuth/JWT/JWKS | Public JWKS read returned 200, two RSA keys and decoded through the existing Swift reader; no tokens recorded | Existing production owner with synthetic signed issuer; open response and authentication failure controls | +| Snapshot metadata/baseline | Upstream schema only; no live signed-in response | Public Core raw completions verify extension tolerance, required/type/null and HTTPS URL constraints | +| Transit sync/snapshot/assets | No new live graph/asset samples | Existing envelope/size/transaction/snapshot/asset regression suites; no format changes | +| Owned native/persistence/output | OCaml producer emitted real generated bytes to the Swift public receiver in an isolated harness; no real account store accessed | Native wire round trip, nested closed decoder, optional/dictionary behavior and account recovery; existing OCaml closed persistence tests | + +Current upstream dev `949a4f0e72cddb2dad34091393bfa09592171944` declares optional non-null UUID tx-id on server tx-log entries; old `015cc9f` does not declare that server field. Both use open Malli maps. This is source evidence, not the screenshot field's observed type or deployed revision: [frozen dev schema](https://github.com/logseq/logseq/blob/949a4f0e72cddb2dad34091393bfa09592171944/deps/db-sync/src/logseq/db_sync/malli_schema.cljs#L10). Journal does not consume tx-id, so its external input reader ignores it. + +The complete build (`@all` and native embed), full regression recheck (259 sync cases), 10 native JSON boundary assertions, account/recovery/event and synthetic signed-Cognito suites, public OCaml-to-Swift wire round trip and production iOS SwiftUI typecheck passed. The initial concurrent full run exposed one existing 20ms/30ms RFC Pong timing failure; both original and restored protocol single-case checks then passed, followed by the full regression recheck. No transport timing or production transport code was changed. The document passes spec-dev-tool; check --all reports three unchanged preexisting invalid historical decisions. + +A preliminary independent read-only review of the uncommitted diff using journal-code-review found no evidenced P1/P2. It does not replace the independent Codex session review of the formal PR snapshot. diff --git a/logseq_db_storage/lib/sync_outbox_store.ml b/logseq_db_storage/lib/sync_outbox_store.ml index 23307a5b..79c87a25 100644 --- a/logseq_db_storage/lib/sync_outbox_store.ml +++ b/logseq_db_storage/lib/sync_outbox_store.ml @@ -33,7 +33,9 @@ let schema db = | _ -> Error "unsupported sync outbox storage format") ;; -type mutation_identity = { mutation_id : string } [@@deriving yojson { strict = false }] +(* These decode-only projections extract indexing metadata. The full persisted + record is admitted by the closed domain codec at its owner. *) +type mutation_identity = { mutation_id : string } [@@deriving of_yojson { strict = false }] type mutation_identity_wire = | Save_block of mutation_identity @@ -44,7 +46,7 @@ type mutation_identity_wire = | Create_journal_page of mutation_identity | Set_task_status of mutation_identity | Clear_task_status of mutation_identity -[@@deriving yojson] +[@@deriving of_yojson] type payload_header = { format_version : int @@ -52,7 +54,7 @@ type payload_header = ; sequence : int ; sync_revision : int } -[@@deriving yojson { strict = false }] +[@@deriving of_yojson { strict = false }] let payload_identity record = match diff --git a/logseq_db_worker/test/test_protocol.ml b/logseq_db_worker/test/test_protocol.ml index 3d642588..083db60d 100644 --- a/logseq_db_worker/test/test_protocol.ml +++ b/logseq_db_worker/test/test_protocol.ml @@ -1,6 +1,56 @@ module T = Logseq_db_worker_test_support.Test_support module P = Logseq_db_worker.Protocol +let closed_internal_config_fields () = + let base = + `Assoc + [ "application_support_directory", `String "/tmp/synthetic-support" + ; ( "target" + , `List + [ `String "Managed_sync" + ; `Assoc [ "base_url", `String "https://example.invalid" ] + ] ) + ; "compatibility_profile", `List [ `String "Logseq_65_33_or_newer" ] + ; "response_budget_bytes", `Int P.maximum_response_bytes + ; "default_page_size", `Int P.default_page_size + ] + in + T.require + (Result.is_ok (Logseq_db_worker.Config.of_yojson base)) + "valid owned Config control was rejected"; + let outer = + match base with + | `Assoc fields -> `Assoc (("future", `Bool true) :: fields) + | _ -> assert false + in + let nested = + match base with + | `Assoc fields -> + `Assoc + (List.map + (fun (key, value) -> + if key = "target" + then + ( key + , `List + [ `String "Managed_sync" + ; `Assoc + [ "base_url", `String "https://example.invalid" + ; "future", `Bool true + ] + ] ) + else key, value) + fields) + | _ -> assert false + in + List.iter + (fun json -> + T.require + (Result.is_error (Logseq_db_worker.Config.of_yojson json)) + "owned internal Config accepted an extra field") + [ outer; nested ] +;; + let page_tree_contract_rejects_retired_scope () = let raw = Yojson.Safe.from_string @@ -934,7 +984,8 @@ let request_tree_boundaries () = let () = T.run "protocol" - [ T.case "tree depth node and root boundaries" request_tree_boundaries + [ T.case "closed internal Config root and nested fields" closed_internal_config_fields + ; T.case "tree depth node and root boundaries" request_tree_boundaries ; T.case "config golden and validation" config_generated_golden_and_validation ; T.case "generated golden and legacy rejection" generated_golden_and_legacy_rejection ; T.case "request domain constraints" request_domain_constraints diff --git a/logseq_sync/lib/effect_runner/protocol/catalog.ml b/logseq_sync/lib/effect_runner/protocol/catalog.ml index bdb03aca..9b78b5ec 100644 --- a/logseq_sync/lib/effect_runner/protocol/catalog.ml +++ b/logseq_sync/lib/effect_runner/protocol/catalog.ml @@ -44,7 +44,7 @@ let schema = function ;; type readiness = { ready : bool option [@key "graph-ready-for-use?"] [@default None] } -[@@deriving yojson { strict = false }] +[@@deriving of_yojson { strict = false }] type graph_dto = { graph_id : Graph_types.Uuid.t [@key "graph-id"] @@ -52,9 +52,9 @@ type graph_dto = ; schema_version : string option [@key "schema-version"] [@default None] ; encrypted : bool option [@key "graph-e2ee?"] [@default None] } -[@@deriving yojson { strict = false }] +[@@deriving of_yojson { strict = false }] -type response = { graphs : Yojson.Safe.t list } [@@deriving yojson { strict = false }] +type response = { graphs : Yojson.Safe.t list } [@@deriving of_yojson { strict = false }] let decode_graph json = bind (readiness_of_yojson json) (fun ready -> diff --git a/logseq_sync/lib/effect_runner/protocol/e2ee.ml b/logseq_sync/lib/effect_runner/protocol/e2ee.ml index b5bf3a3b..aa038281 100644 --- a/logseq_sync/lib/effect_runner/protocol/e2ee.ml +++ b/logseq_sync/lib/effect_runner/protocol/e2ee.ml @@ -40,15 +40,15 @@ let bounded_string field value = type graph_key_response_dto = { encrypted_aes_key : string [@key "encrypted-aes-key"] ; encrypted_aes_key_base64 : string option - [@key "encrypted-aes-key-base64"] [@default None] + [@key "encrypted-aes-key-base64"] [@default None] } -[@@deriving yojson] +[@@deriving of_yojson { strict = false }] type user_keys_response_dto = { public_key : string [@key "public-key"] ; encrypted_private_key : string [@key "encrypted-private-key"] } -[@@deriving yojson] +[@@deriving of_yojson { strict = false }] let graph_key_response source = try @@ -56,7 +56,10 @@ let graph_key_response source = (Logseq_db_types.Json_codec.decode graph_key_response_dto_of_yojson (Yojson.Safe.from_string source)) - (fun dto -> bounded_string "encrypted-aes-key" dto.encrypted_aes_key) + (fun dto -> + (* The optional compatibility field is validated by PPX but not consumed. *) + let _ = dto.encrypted_aes_key_base64 in + bounded_string "encrypted-aes-key" dto.encrypted_aes_key) with | Yojson.Json_error _ -> Error "E2EE response must be valid JSON" ;; diff --git a/logseq_sync/lib/pure_reducer/core.ml b/logseq_sync/lib/pure_reducer/core.ml index 9be57439..886150ed 100644 --- a/logseq_sync/lib/pure_reducer/core.ml +++ b/logseq_sync/lib/pure_reducer/core.ml @@ -1019,7 +1019,7 @@ type snapshot_uri_response = { ok : bool ; url : string } -[@@deriving yojson { strict = false }] +[@@deriving of_yojson { strict = false }] let decode_snapshot_uri source = try diff --git a/logseq_sync/lib/pure_reducer/sync_protocol.ml b/logseq_sync/lib/pure_reducer/sync_protocol.ml index fb6c444c..7d9ed4a9 100644 --- a/logseq_sync/lib/pure_reducer/sync_protocol.ml +++ b/logseq_sync/lib/pure_reducer/sync_protocol.ml @@ -163,8 +163,6 @@ let error_to_string (error : codec_error) = ;; module Wire = struct - type header = { kind : string [@key "type"] } [@@deriving yojson { strict = false }] - type client_transaction = { tx : string ; tx_id : Logseq_db_types.Graph_types.Uuid.t option [@key "tx-id"] [@default None] @@ -279,12 +277,74 @@ module Wire = struct ; message : string } [@@deriving yojson] +end - type baseline = - { kind : string [@key "type"] - ; t : int64 +(* External input has readers only; owned output keeps the closed Wire codecs. *) +module External_input = struct + type header = { kind : string [@key "type"] } [@@deriving of_yojson { strict = false }] + + type pull_transaction = + { t : int64 + ; tx : string + ; outliner_op : string option [@key "outliner-op"] [@default None] } - [@@deriving yojson { strict = false }] + [@@deriving of_yojson { strict = false }] + + type user_presence = + { user_id : string [@key "user-id"] + ; email : string option [@default None] + ; username : string option [@default None] + ; name : string option [@default None] + } + [@@deriving of_yojson { strict = false }] + + type server_hello = + { t : int64 + ; checksum : string option [@default None] + } + [@@deriving of_yojson { strict = false }] + + type server_online_users = { online_users : user_presence list [@key "online-users"] } + [@@deriving of_yojson { strict = false }] + + type server_presence = + { user_id : string [@key "user-id"] + ; editing_block_uuid : string option [@key "editing-block-uuid"] + } + [@@deriving of_yojson { strict = false }] + + type server_pull_ok = + { t : int64 + ; checksum : string option [@default None] + ; txs : pull_transaction list + } + [@@deriving of_yojson { strict = false }] + + type server_tx_batch_ok = + { t : int64 + ; checksum : string option [@default None] + } + [@@deriving of_yojson { strict = false }] + + type server_changed = { t : int64 } [@@deriving of_yojson { strict = false }] + + type server_tx_reject = + { reason : string + ; t : int64 option [@default None] + ; checksum : string option [@default None] + ; success_tx_ids : Logseq_db_types.Graph_types.Uuid.t list + [@key "success-tx-ids"] [@default []] + ; failed_tx_id : Logseq_db_types.Graph_types.Uuid.t option + [@key "failed-tx-id"] [@default None] + ; missing_block_uuids : Logseq_db_types.Graph_types.Uuid.t list + [@key "missing-block-uuids"] [@default []] + ; error_detail : string option [@key "error-detail"] [@default None] + ; data : string option [@default None] + } + [@@deriving of_yojson { strict = false }] + + type server_error = { message : string } [@@deriving of_yojson { strict = false }] + type baseline = { t : int64 } [@@deriving of_yojson { strict = false }] end (* Structural checks preserve the public safe diagnostics. They never construct @@ -330,7 +390,11 @@ let check_fields context path allowed fields = | _ :: _ -> error context path (Unexpected_fields unexpected)) ;; -let rec validate context path shape json = +type object_policy = + | Closed + | Open + +let rec validate ?(object_policy = Closed) context path shape json = let invalid expected = error context path (Invalid_field { expected }) in match shape, json with | Text maximum, `String value | Nullable_text maximum, `String value -> @@ -383,14 +447,23 @@ let rec validate context path shape json = let rec loop index = function | [] -> Ok () | value :: rest -> - let* () = validate context (path @ [ string_of_int index ]) item value in + let* () = + validate ~object_policy context (path @ [ string_of_int index ]) item value + in loop (index + 1) rest in loop 0 values | Array _, _ -> invalid "array" | Object schema, `Assoc fields -> let* () = - check_fields context path (List.map (fun (name, _, _) -> name) schema) fields + match object_policy with + | Closed -> + check_fields context path (List.map (fun (name, _, _) -> name) schema) fields + | Open -> + (match duplicate_names (List.map fst fields) with + | [] -> Ok () + | _ :: _ -> + error context path (Invalid_field { expected = "unique object fields" })) in let rec loop = function | [] -> Ok () @@ -399,7 +472,7 @@ let rec validate context path shape json = match List.assoc_opt name fields with | None when is_required -> error context path (Missing_field name) | None -> Ok () - | Some json -> validate context (path @ [ name ]) shape json + | Some json -> validate ~object_policy context (path @ [ name ]) shape json in loop rest in @@ -445,8 +518,10 @@ let rejection_shape = ] ;; -let decode_dto context schema decoder json = - let* () = validate context [] (Object (required "type" metadata :: schema)) json in +let decode_dto ?(object_policy = Closed) context schema decoder json = + let* () = + validate ~object_policy context [] (Object (required "type" metadata :: schema)) json + in match Logseq_db_types.Json_codec.decode decoder json with | Ok dto -> Ok dto | Error _ -> error context [] (Invalid_field { expected = "valid protocol fields" }) @@ -459,11 +534,11 @@ let client_transaction (value : Wire.client_transaction) : Client.transaction = { tx = value.tx; tx_id = value.tx_id; outliner_op = value.outliner_op } ;; -let pull_transaction (value : Wire.pull_transaction) : Server.pull_transaction = +let pull_transaction (value : External_input.pull_transaction) : Server.pull_transaction = { t = cursor value.t; tx = value.tx; outliner_op = value.outliner_op } ;; -let user_presence (value : Wire.user_presence) : user_presence = +let user_presence (value : External_input.user_presence) : user_presence = { user_id = value.user_id ; email = value.email ; username = value.username @@ -560,13 +635,16 @@ let decode_client_fields context message_type json = ;; let decode_server_fields context message_type json = + let decode_dto context schema decoder json = + decode_dto ~object_policy:Open context schema decoder json + in match message_type with | "hello" -> let* dto = decode_dto context [ required "t" Cursor; optional "checksum" Checksum ] - Wire.server_hello_of_yojson + External_input.server_hello_of_yojson json in Ok (Server.Hello { t = cursor dto.t; checksum = dto.checksum }) @@ -575,7 +653,7 @@ let decode_server_fields context message_type json = decode_dto context [ required "online-users" (Array user_presence_shape) ] - Wire.server_online_users_of_yojson + External_input.server_online_users_of_yojson json in Ok (Server.Online_users { online_users = List.map user_presence dto.online_users }) @@ -584,7 +662,7 @@ let decode_server_fields context message_type json = decode_dto context [ required "user-id" metadata; required "editing-block-uuid" nullable_metadata ] - Wire.server_presence_of_yojson + External_input.server_presence_of_yojson json in Ok @@ -598,7 +676,7 @@ let decode_server_fields context message_type json = ; optional "checksum" Checksum ; required "txs" (Array pull_transaction_shape) ] - Wire.server_pull_ok_of_yojson + External_input.server_pull_ok_of_yojson json in Ok @@ -612,17 +690,23 @@ let decode_server_fields context message_type json = decode_dto context [ required "t" Cursor; optional "checksum" Checksum ] - Wire.server_tx_batch_ok_of_yojson + External_input.server_tx_batch_ok_of_yojson json in Ok (Server.Tx_batch_ok { t = cursor dto.t; checksum = dto.checksum }) | "changed" -> let* dto = - decode_dto context [ required "t" Cursor ] Wire.server_changed_of_yojson json + decode_dto + context + [ required "t" Cursor ] + External_input.server_changed_of_yojson + json in Ok (Server.Changed { t = cursor dto.t }) | "tx/reject" -> - let* dto = decode_dto context rejection_shape Wire.server_tx_reject_of_yojson json in + let* dto = + decode_dto context rejection_shape External_input.server_tx_reject_of_yojson json + in let* rejection = validate_rejection context @@ -638,11 +722,15 @@ let decode_server_fields context message_type json = in Ok (Server.Tx_reject rejection) | "pong" -> - let* _ = decode_dto context [] Wire.server_pong_of_yojson json in + let* _ = decode_dto context [] External_input.header_of_yojson json in Ok Server.Pong | "error" -> let* dto = - decode_dto context [ required "message" metadata ] Wire.server_error_of_yojson json + decode_dto + context + [ required "message" metadata ] + External_input.server_error_of_yojson + json in Ok (Server.Error { message = dto.message }) | unsupported -> error context [] (Unsupported_message_type unsupported) @@ -662,7 +750,7 @@ let decode ?maximum direction decode_fields wire = | None -> error context [] (Missing_field "type") | Some json -> validate context [ "type" ] metadata json in - (match Wire.header_of_yojson json with + (match External_input.header_of_yojson json with | Ok header -> decode_fields { context with message_type = Some header.kind } header.kind json | Error _ -> @@ -687,7 +775,9 @@ let decode_snapshot_baseline = | None -> error context [] (Missing_field "t") | Some json -> validate context [ "t" ] Cursor json in - (match Logseq_db_types.Json_codec.decode Wire.baseline_of_yojson json with + (match + Logseq_db_types.Json_codec.decode External_input.baseline_of_yojson json + with | Ok dto -> Ok (cursor dto.t) | Error _ -> error context [] (Invalid_field { expected = "snapshot baseline" })) @@ -775,6 +865,31 @@ let encode_client_message message = encode decode_client_message (client_message_json message) ;; +(* The external reader is never used as the field-set validator for output. *) +let validate_server_output json = + let context = { direction = Server; message_type = None } in + let check decoder = + match Logseq_db_types.Json_codec.decode decoder json with + | Ok _ -> Ok () + | Error _ -> + error context [] (Invalid_field { expected = "closed server output fields" }) + in + match External_input.header_of_yojson json with + | Ok { kind = "hello" } -> check Wire.server_hello_of_yojson + | Ok { kind = "online-users" } -> check Wire.server_online_users_of_yojson + | Ok { kind = "presence" } -> check Wire.server_presence_of_yojson + | Ok { kind = "pull/ok" } -> check Wire.server_pull_ok_of_yojson + | Ok { kind = "tx/batch/ok" } -> check Wire.server_tx_batch_ok_of_yojson + | Ok { kind = "changed" } -> check Wire.server_changed_of_yojson + | Ok { kind = "tx/reject" } -> check Wire.server_tx_reject_of_yojson + | Ok { kind = "pong" } -> check Wire.server_pong_of_yojson + | Ok { kind = "error" } -> check Wire.server_error_of_yojson + | Ok { kind } -> error context [] (Unsupported_message_type kind) + | Error _ -> error context [] (Invalid_field { expected = "message discriminator" }) +;; + let encode_server_message message = - encode decode_server_message (server_message_json message) + let json = server_message_json message in + let* () = validate_server_output json in + encode decode_server_message json ;; diff --git a/logseq_sync/test/core_contract.ml b/logseq_sync/test/core_contract.ml index 50f1db94..b7ab164f 100644 --- a/logseq_sync/test/core_contract.ml +++ b/logseq_sync/test/core_contract.ml @@ -5196,3 +5196,61 @@ let scenarios = scenarios @ ; Alcotest.test_case "E2EE invalidated challenge is not advertised" `Quick e2ee_invalidated_challenge_is_not_advertised ] ;; + + +let snapshot_metadata_open_fields_keep_declared_constraints () = + List.iter + (fun ((body : string), success) -> + let requested, _ = begin_snapshot_bootstrap () in + let metadata = + List.find_map + (function + | Core.Run (Core.Request (ticket, Core.Fetch_snapshot_baseline _)) -> + Some + (Core.step + requested.next + (Core.Runner_completed + (Core.Completion + ( ticket + , Ok {|{"type":"pull/ok","t":1,"future":{"nested":true}}|} )))) + | _ -> None) + requested.effects + |> Option.get + in + let downloaded = + List.find_map + (function + | Core.Run (Core.Request (ticket, Core.Fetch_snapshot_metadata _)) -> + Some + (Core.step + metadata.next + (Core.Runner_completed (Core.Completion (ticket, Ok body)))) + | _ -> None) + metadata.effects + |> Option.get + in + Alcotest.(check bool) + "public Core owns metadata decoding and download decision" + success + (List.exists + (function + | Core.Run (Core.Request (_, Core.Download_snapshot _)) -> true + | _ -> false) + downloaded.effects)) + [ ( {|{"ok":true,"url":"https://example.com/snapshot","key":"synthetic","future":{"nested":true}}|} + , true ) + ; {|{"ok":true,"future":true}|}, false + ; {|{"ok":true,"url":null,"future":true}|}, false + ; {|{"ok":"true","url":"https://example.com/snapshot","future":true}|}, false + ; {|{"ok":true,"url":"http://example.com/snapshot","future":true}|}, false + ] +;; + +let scenarios = + scenarios + @ [ Alcotest.test_case + "snapshot metadata open fields keep declared constraints" + `Quick + snapshot_metadata_open_fields_keep_declared_constraints + ] +;; diff --git a/logseq_sync/test/sync_protocol_contract.ml b/logseq_sync/test/sync_protocol_contract.ml index 82527328..4bccfa94 100644 --- a/logseq_sync/test/sync_protocol_contract.ml +++ b/logseq_sync/test/sync_protocol_contract.ml @@ -215,36 +215,7 @@ let test_unknown_fields_are_strict_and_safe () = Alcotest.bool "nested error omits transaction" false - (contains (Protocol.error_to_string nested) "secret tx"); - let pulled_transaction = - Protocol.decode_server_message - {|{"type":"pull/ok","t":1,"txs":[{"t":1,"tx":"secret tx","extra":1}]}|} - |> require_error "unknown pulled transaction field" - in - Alcotest.(check (list string)) - "pulled transaction path is precise" - [ "txs"; "0" ] - pulled_transaction.path; - Alcotest.check - Alcotest.bool - "pulled transaction field is named" - true - (pulled_transaction.kind = Protocol.Unexpected_fields [ "extra" ]); - Alcotest.check - Alcotest.bool - "pulled transaction error omits transaction" - false - (contains (Protocol.error_to_string pulled_transaction) "secret tx"); - let presence = - Protocol.decode_server_message - {|{"type":"online-users","online-users":[{"user-id":"u1","future":true}]}|} - |> require_error "unknown online user field" - in - Alcotest.check Alcotest.bool "direction is server" true (presence.direction = Server); - Alcotest.(check (list string)) - "online-user path is precise" - [ "online-users"; "0" ] - presence.path + (contains (Protocol.error_to_string nested) "secret tx") ;; let test_decode_errors_are_structured () = @@ -388,8 +359,165 @@ let test_cursor_wire_range () = (Client.Pull { since = Some (server_cursor_of_int 9007199254740992) }))) ;; +(* Core receives typed Server.message values, so raw extension acceptance is + owned by this public codec boundary rather than a fabricated reducer event. *) +let external_server_cases = + [ {|{"type":"hello","t":0}|} + ; {|{"type":"online-users","online-users":[{"user-id":"u","future-user":{"nested":true}}]}|} + ; {|{"type":"presence","user-id":"u","editing-block-uuid":null}|} + ; {|{"type":"pull/ok","t":1,"txs":[{"t":1,"tx":"synthetic transit","outliner-op":null,"future-entry":{"nested":true}}]}|} + ; {|{"type":"tx/batch/ok","t":1}|} + ; {|{"type":"changed","t":1}|} + ; {|{"type":"tx/reject","reason":"stale","t":1}|} + ; {|{"type":"tx/reject","reason":"snapshot upload in progress","t":1}|} + ; {|{"type":"tx/reject","reason":"empty tx data"}|} + ; {|{"type":"tx/reject","reason":"invalid tx"}|} + ; {|{"type":"tx/reject","reason":"invalid t-before"}|} + ; {|{"type":"tx/reject","reason":"db transact failed","t":1}|} + ; {|{"type":"pong"}|} + ; {|{"type":"error","message":"synthetic server error"}|} + ] +;; + +let test_external_server_extensions_are_open () = + List.iter + (fun source -> + let json = Yojson.Safe.from_string source in + let extended = + match json with + | `Assoc fields -> + `Assoc + (("future-top", `Assoc [ "nested", `List [ `Null; `Bool true ] ]) :: fields) + | _ -> assert false + in + let input = Yojson.Safe.to_string extended in + let message = + Protocol.decode_server_message input |> require_ok "open external server input" + in + let canonical = + Protocol.encode_server_message message |> require_ok "closed server output" + in + let without_extension = Yojson.Safe.from_string canonical in + let fields = + match without_extension with + | `Assoc f -> f + | _ -> assert false + in + Alcotest.(check bool) + "external extension does not enter owned output" + false + (List.mem_assoc "future-top" fields)) + external_server_cases; + List.iter + (fun ignored -> + let json = + `Assoc + [ "type", `String "pull/ok" + ; "t", `Int 1 + ; ( "txs" + , `List + [ `Assoc + [ "t", `Int 1; "tx", `String "synthetic transit"; "tx-id", ignored ] + ] ) + ] + in + let message = + Protocol.decode_server_message (Yojson.Safe.to_string json) + |> require_ok "unused real-world tx-id extension" + in + let wire = + Protocol.encode_server_message message |> require_ok "re-encode owned pull" + in + Alcotest.(check bool) + "unused tx-id is not reflected in output" + false + (contains wire "tx-id")) + [ `Null; `String "synthetic-id"; `Int 1; `Bool true; `List []; `Assoc [] ] +;; + +let test_external_declared_fields_remain_validated () = + List.iter + (fun source -> + let error = + Protocol.decode_server_message source + |> require_error "declared external contract violation" + in + Alcotest.(check bool) + "extensions never bypass declared-field validation" + false + (match error.kind with + | Protocol.Unexpected_fields _ -> true + | _ -> false)) + [ {|{"type":"changed","future":true}|} + ; {|{"type":"changed","t":"1","future":true}|} + ; {|{"type":"changed","t":null,"future":true}|} + ; {|{"type":"pull/ok","t":1,"txs":[{"t":1,"future":true}],"future":true}|} + ; {|{"type":"pull/ok","t":1,"txs":[{"t":1,"tx":null,"future":true}]}|} + ; {|{"type":"online-users","online-users":[{"user-id":null,"future":true}]}|} + ; {|{"type":"presence","user-id":"u","future":true}|} + ; {|{"type":"hello","t":1,"checksum":null,"future":true}|} + ; {|{"type":"tx/reject","reason":"unrecognized","future":true}|} + ; {|{"type":"tx/reject","reason":"stale","t":1,"failed-tx-id":"not-a-uuid","future":true}|} + ; {|{"type":"future-event","future":true}|} + ; {|{"type":"changed","t":1,"t":2,"future":true}|} + ]; + let missing = + Protocol.decode_server_message {|{"type":"changed","future":true}|} + |> require_error "missing required cursor remains classified" + in + Alcotest.(check bool) + "required field takes precedence over extras" + true + (missing.kind = Protocol.Missing_field "t") +;; + +let test_outgoing_wire_contracts_are_closed () = + List.iter + (fun (message, expected) -> + let wire = + Protocol.encode_client_message message |> require_ok "outgoing contract" + in + Alcotest.(check bool) + "generated output has exactly the contract fields" + true + (Yojson.Safe.equal + (Yojson.Safe.from_string wire) + (Yojson.Safe.from_string expected)); + let extended = + match Yojson.Safe.from_string wire with + | `Assoc fields -> `Assoc (("future", `Bool true) :: fields) + | _ -> assert false + in + ignore + (Protocol.decode_client_message (Yojson.Safe.to_string extended) + |> require_error "closed owned client wire")) + [ Client.Hello { client = "synthetic" }, {|{"type":"hello","client":"synthetic"}|} + ; Client.Presence { editing_block_uuid = None }, {|{"type":"presence"}|} + ; Client.Pull { since = None }, {|{"type":"pull"}|} + ; ( Client.Tx_batch + { client_revision = None + ; t_before = server_cursor_of_int 0 + ; txs = [ { tx = "synthetic"; tx_id = None; outliner_op = None } ] + } + , {|{"type":"tx/batch","t-before":0,"txs":[{"tx":"synthetic"}]}|} ) + ; Client.Ping, {|{"type":"ping"}|} + ] +;; + let scenarios = [ Alcotest.test_case + "external server extensions are open" + `Quick + test_external_server_extensions_are_open + ; Alcotest.test_case + "external declared fields remain validated" + `Quick + test_external_declared_fields_remain_validated + ; Alcotest.test_case + "outgoing JSON contracts are closed" + `Quick + test_outgoing_wire_contracts_are_closed + ; Alcotest.test_case "numeric cursor wire safe integer range" `Quick test_cursor_wire_range @@ -406,7 +534,7 @@ let scenarios = `Quick test_all_rejection_shapes_round_trip ; Alcotest.test_case - "unknown fields are strict and safe" + "owned client fields are strict and safe" `Quick test_unknown_fields_are_strict_and_safe ; Alcotest.test_case diff --git a/logseq_sync/test/transport_contract.ml b/logseq_sync/test/transport_contract.ml index 8eb753f5..78406a22 100644 --- a/logseq_sync/test/transport_contract.ml +++ b/logseq_sync/test/transport_contract.ml @@ -4037,7 +4037,7 @@ let test_unlock_response_structure ~expected ~success body () = let scenarios=scenarios @ [ Alcotest.test_case "unlock shape object extra secret field" `Quick - (test_unlock_response_structure ~success:false + (test_unlock_response_structure ~success:true ~expected:"json_kind=object known_fields=1 unknown_fields=1 encrypted_key_present=true" {|{"encrypted-aes-key":"LOG_SECRET_WRAPPED","LOG_SECRET_UNKNOWN_FIELD_NAME":"LOG_SECRET_BODY"}|}) ; Alcotest.test_case "unlock shape array" `Quick @@ -4086,9 +4086,131 @@ let scenarios = scenarios @ (test_unlock_response_structure ~success:false ~expected:"base64_field_kind=number" {|{"encrypted-aes-key":"LOG_SECRET_WRAPPED","encrypted-aes-key-base64":7}|}) - ; Alcotest.test_case "E2EE DTO retains unrelated field strictness" `Quick - (test_unlock_response_structure ~success:false + ; Alcotest.test_case "E2EE DTO ignores unrelated external fields" `Quick + (test_unlock_response_structure ~success:true ~expected:"base64_field_kind=string" {|{"encrypted-aes-key":"LOG_SECRET_WRAPPED","encrypted-aes-key-base64":"LOG_SECRET_BODY","LOG_SECRET_UNKNOWN_FIELD_NAME":false}|}) ] ;; + + +let test_external_http_contract kind ~success body () = + ignore + (with_peer + [ case (unlock_json_wire body) ] + (fun ~environment ~sw ~support ~origin -> + let initial = + Core.config ~managed_sync_origin:origin ~limits:(Core_contract.limits ()) + |> Result.get_ok + |> Core.initial + |> Result.get_ok + in + let request = + match kind with + | `Catalog -> + let auth = + Core.step + initial + (Core.Account_authenticated { user_id = Some "synthetic-user" }) + in + List.find_map + (function + | Core.Run (Core.Request (_, Core.Fetch_catalog _) as request) -> + Some request + | _ -> None) + auth.effects + |> Option.get + | `Graph_key -> snd (unlock_classification_effect origin) + | `User_keys -> + let keys, _ = unlock_classification_effect origin in + let private_keys = + List.find_map + (function + | Core.Run (Core.Request (ticket, Core.Fetch_e2ee_graph_key _)) -> + Some + (Core.step + keys.next + (Core.Runner_completed + (Core.Completion (ticket, Ok "synthetic wrapped key")))) + | _ -> None) + keys.effects + |> Option.get + in + List.find_map + (function + | Core.Run (Core.Request (_, Core.Fetch_e2ee_user_keys _) as request) -> + Some request + | _ -> None) + private_keys.effects + |> Option.get + in + let posted = ref [] + and invalidations = ref 0 in + let t = runner ~environment ~sw ~support ~posted ~invalidations () in + Fun.protect + ~finally:(fun () -> Runner.shutdown t) + (fun () -> + Runner.submit t request; + wait (Eio.Stdenv.clock environment) (fun () -> !posted <> []); + Alcotest.(check bool) + "actual HTTP JSON decoder completion" + success + (List.exists + (function + | Core.Runner_completed (Core.Completion (_, result)) -> + Result.is_ok result + | _ -> false) + !posted))) + : Yojson.Basic.t) +;; + +let scenarios = + scenarios + @ [ Alcotest.test_case + "HTTP graph key ignores external extensions" + `Quick + (test_external_http_contract + `Graph_key + ~success:true + {|{"encrypted-aes-key":"synthetic","future":{"nested":null}}|}) + ; Alcotest.test_case + "HTTP user keys ignore external extensions" + `Quick + (test_external_http_contract + `User_keys + ~success:true + {|{"public-key":"synthetic","encrypted-private-key":"synthetic","future":{"nested":[1,null]}}|}) + ; Alcotest.test_case + "HTTP catalog ignores nested external extensions" + `Quick + (test_external_http_contract + `Catalog + ~success:true + {|{"graphs":[{"graph-id":"11111111-1111-4111-8111-111111111111","graph-name":"synthetic","graph-ready-for-use?":true,"schema-version":null,"future":{"nested":true}}],"future":[1,null]}|}) + ; Alcotest.test_case + "HTTP graph key validates declared fields with extras" + `Quick + (test_external_http_contract + `Graph_key + ~success:false + {|{"encrypted-aes-key":null,"future":true}|}) + ; Alcotest.test_case + "HTTP graph key still requires declared fields" + `Quick + (test_external_http_contract `Graph_key ~success:false {|{"future":true}|}) + ; Alcotest.test_case + "HTTP user keys validate declared fields with extras" + `Quick + (test_external_http_contract + `User_keys + ~success:false + {|{"public-key":"synthetic","encrypted-private-key":false,"future":true}|}) + ; Alcotest.test_case + "HTTP catalog validates nested declared fields" + `Quick + (test_external_http_contract + `Catalog + ~success:false + {|{"graphs":[{"graph-id":"11111111-1111-4111-8111-111111111111","graph-name":null,"graph-ready-for-use?":true,"future":true}]}|}) + ] +;; diff --git a/swift/JournalCognitoNative.swift b/swift/JournalCognitoNative.swift index 3a20c806..4c9e7ca5 100644 --- a/swift/JournalCognitoNative.swift +++ b/swift/JournalCognitoNative.swift @@ -37,7 +37,7 @@ private final class JournalCognitoKeychain: Sendable { if status == errSecItemNotFound { return nil } guard status == errSecSuccess, let data = item as? Data, data.count <= 131072 else { throw JournalCognitoError.storage } - return try JSONDecoder().decode(JournalCognitoTokens.self, from: data) + return try JournalInternalJSON.decode(JournalCognitoTokens.self, from: data) } } func save(_ tokens: JournalCognitoTokens) async throws { diff --git a/swift/JournalE2EECrypto.swift b/swift/JournalE2EECrypto.swift index 65b4da1e..fe06b5aa 100644 --- a/swift/JournalE2EECrypto.swift +++ b/swift/JournalE2EECrypto.swift @@ -1119,7 +1119,7 @@ public func logseqJournalCryptoJSON( let requestPointer, let data = String(cString: requestPointer).data(using: .utf8) else { throw JournalE2EECryptoError.invalidRequest } - let request = try JSONDecoder().decode(JournalCryptoRequest.self, from: data) + let request = try JournalInternalJSON.decode(JournalCryptoRequest.self, from: data) if ProcessInfo.processInfo.environment["JOURNAL_UNLOCK_DIAGNOSTICS"] == "1" { trace = JournalUnlockTrace(request) JournalUnlockTrace.current = trace diff --git a/swift/JournalExtensions.swift b/swift/JournalExtensions.swift index abd3a4c8..9cae8fa6 100644 --- a/swift/JournalExtensions.swift +++ b/swift/JournalExtensions.swift @@ -96,7 +96,7 @@ enum JournalExtensionFingerprint { _ type: Properties.Type, context: LUIAppleExtensionViewContext ) -> Properties? { guard case let .string(json) = context.property("payload"), - let value = try? JSONDecoder().decode(type, from: Data(json.utf8)) + let value = try? JournalInternalJSON.decode(type, from: Data(json.utf8)) else { return nil } return value } diff --git a/swift/JournalInternalJSON.swift b/swift/JournalInternalJSON.swift new file mode 100644 index 00000000..7d36edca --- /dev/null +++ b/swift/JournalInternalJSON.swift @@ -0,0 +1,146 @@ +import Foundation + +/// Owned JSON uses synthesized CodingKeys as its closed schema, recursively. +/// External service readers continue using their ordinary JSONDecoder. +enum JournalInternalJSON { + static func decode(_ type: Value.Type, from data: Data) throws -> Value { + let decoder = JSONDecoder() + if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { + return try decoder.decode(type, from: data) + } + return try decoder.decode(OwnedValue.self, from: data).value + } +} + +private struct AnyField: CodingKey { + let stringValue: String + let intValue: Int? + init?(stringValue: String) { self.stringValue = stringValue; intValue = nil } + init?(intValue: Int) { stringValue = String(intValue); self.intValue = intValue } +} + +private struct OwnedValue: Decodable { + let value: Value + init(from decoder: Decoder) throws { value = try Value(from: OwnedDecoder(base: decoder)) } +} + +private struct OwnedDecoder: Decoder { + let base: Decoder + var codingPath: [CodingKey] { base.codingPath } + var userInfo: [CodingUserInfoKey: Any] { base.userInfo } + + func container(keyedBy type: Key.Type) throws -> KeyedDecodingContainer { + let fields = try base.container(keyedBy: AnyField.self).allKeys + guard fields.allSatisfy({ Key(stringValue: $0.stringValue) != nil }) else { + throw DecodingError.dataCorrupted(.init(codingPath: codingPath, + debugDescription: "Unexpected field in owned JSON object")) + } + return KeyedDecodingContainer(OwnedKeyed(base: try base.container(keyedBy: type))) + } + func unkeyedContainer() throws -> UnkeyedDecodingContainer { + OwnedUnkeyed(base: try base.unkeyedContainer()) + } + func singleValueContainer() throws -> SingleValueDecodingContainer { + OwnedSingle(base: try base.singleValueContainer(), decoder: self) + } +} + +private struct OwnedKeyed: KeyedDecodingContainerProtocol { + let base: KeyedDecodingContainer + var codingPath: [CodingKey] { base.codingPath } + var allKeys: [Key] { base.allKeys } + func contains(_ key: Key) -> Bool { base.contains(key) } + func decodeNil(forKey key: Key) throws -> Bool { try base.decodeNil(forKey: key) } + func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool { try base.decode(type, forKey: key) } + func decode(_ type: String.Type, forKey key: Key) throws -> String { try base.decode(type, forKey: key) } + func decode(_ type: Double.Type, forKey key: Key) throws -> Double { try base.decode(type, forKey: key) } + func decode(_ type: Float.Type, forKey key: Key) throws -> Float { try base.decode(type, forKey: key) } + func decode(_ type: Int.Type, forKey key: Key) throws -> Int { try base.decode(type, forKey: key) } + func decode(_ type: Int8.Type, forKey key: Key) throws -> Int8 { try base.decode(type, forKey: key) } + func decode(_ type: Int16.Type, forKey key: Key) throws -> Int16 { try base.decode(type, forKey: key) } + func decode(_ type: Int32.Type, forKey key: Key) throws -> Int32 { try base.decode(type, forKey: key) } + func decode(_ type: Int64.Type, forKey key: Key) throws -> Int64 { try base.decode(type, forKey: key) } + func decode(_ type: UInt.Type, forKey key: Key) throws -> UInt { try base.decode(type, forKey: key) } + func decode(_ type: UInt8.Type, forKey key: Key) throws -> UInt8 { try base.decode(type, forKey: key) } + func decode(_ type: UInt16.Type, forKey key: Key) throws -> UInt16 { try base.decode(type, forKey: key) } + func decode(_ type: UInt32.Type, forKey key: Key) throws -> UInt32 { try base.decode(type, forKey: key) } + func decode(_ type: UInt64.Type, forKey key: Key) throws -> UInt64 { try base.decode(type, forKey: key) } + func decode(_ type: Value.Type, forKey key: Key) throws -> Value { + // Preserve Foundation's specialized value decoding (not record schemas). + if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { + return try base.decode(type, forKey: key) + } + return try base.decode(OwnedValue.self, forKey: key).value + } + func nestedContainer(keyedBy type: Nested.Type, forKey key: Key) throws -> KeyedDecodingContainer { + try OwnedDecoder(base: base.superDecoder(forKey: key)).container(keyedBy: type) + } + func nestedUnkeyedContainer(forKey key: Key) throws -> UnkeyedDecodingContainer { + try OwnedDecoder(base: base.superDecoder(forKey: key)).unkeyedContainer() + } + func superDecoder() throws -> Decoder { OwnedDecoder(base: try base.superDecoder()) } + func superDecoder(forKey key: Key) throws -> Decoder { OwnedDecoder(base: try base.superDecoder(forKey: key)) } +} + +private struct OwnedUnkeyed: UnkeyedDecodingContainer { + var base: UnkeyedDecodingContainer + var codingPath: [CodingKey] { base.codingPath } + var count: Int? { base.count } + var isAtEnd: Bool { base.isAtEnd } + var currentIndex: Int { base.currentIndex } + mutating func decodeNil() throws -> Bool { try base.decodeNil() } + mutating func decode(_ type: Bool.Type) throws -> Bool { try base.decode(type) } + mutating func decode(_ type: String.Type) throws -> String { try base.decode(type) } + mutating func decode(_ type: Double.Type) throws -> Double { try base.decode(type) } + mutating func decode(_ type: Float.Type) throws -> Float { try base.decode(type) } + mutating func decode(_ type: Int.Type) throws -> Int { try base.decode(type) } + mutating func decode(_ type: Int8.Type) throws -> Int8 { try base.decode(type) } + mutating func decode(_ type: Int16.Type) throws -> Int16 { try base.decode(type) } + mutating func decode(_ type: Int32.Type) throws -> Int32 { try base.decode(type) } + mutating func decode(_ type: Int64.Type) throws -> Int64 { try base.decode(type) } + mutating func decode(_ type: UInt.Type) throws -> UInt { try base.decode(type) } + mutating func decode(_ type: UInt8.Type) throws -> UInt8 { try base.decode(type) } + mutating func decode(_ type: UInt16.Type) throws -> UInt16 { try base.decode(type) } + mutating func decode(_ type: UInt32.Type) throws -> UInt32 { try base.decode(type) } + mutating func decode(_ type: UInt64.Type) throws -> UInt64 { try base.decode(type) } + mutating func decode(_ type: Value.Type) throws -> Value { + if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { + return try base.decode(type) + } + return try base.decode(OwnedValue.self).value + } + mutating func nestedContainer(keyedBy type: Nested.Type) throws -> KeyedDecodingContainer { + try OwnedDecoder(base: base.superDecoder()).container(keyedBy: type) + } + mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { + try OwnedDecoder(base: base.superDecoder()).unkeyedContainer() + } + mutating func superDecoder() throws -> Decoder { OwnedDecoder(base: try base.superDecoder()) } +} + +private struct OwnedSingle: SingleValueDecodingContainer { + let base: SingleValueDecodingContainer + let decoder: OwnedDecoder + var codingPath: [CodingKey] { base.codingPath } + func decodeNil() -> Bool { base.decodeNil() } + func decode(_ type: Bool.Type) throws -> Bool { try base.decode(type) } + func decode(_ type: String.Type) throws -> String { try base.decode(type) } + func decode(_ type: Double.Type) throws -> Double { try base.decode(type) } + func decode(_ type: Float.Type) throws -> Float { try base.decode(type) } + func decode(_ type: Int.Type) throws -> Int { try base.decode(type) } + func decode(_ type: Int8.Type) throws -> Int8 { try base.decode(type) } + func decode(_ type: Int16.Type) throws -> Int16 { try base.decode(type) } + func decode(_ type: Int32.Type) throws -> Int32 { try base.decode(type) } + func decode(_ type: Int64.Type) throws -> Int64 { try base.decode(type) } + func decode(_ type: UInt.Type) throws -> UInt { try base.decode(type) } + func decode(_ type: UInt8.Type) throws -> UInt8 { try base.decode(type) } + func decode(_ type: UInt16.Type) throws -> UInt16 { try base.decode(type) } + func decode(_ type: UInt32.Type) throws -> UInt32 { try base.decode(type) } + func decode(_ type: UInt64.Type) throws -> UInt64 { try base.decode(type) } + func decode(_ type: Value.Type) throws -> Value { + if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { + return try base.decode(type) + } + return try Value(from: decoder) + } +} diff --git a/swift/JournalList.swift b/swift/JournalList.swift index c438bfeb..07ecad15 100644 --- a/swift/JournalList.swift +++ b/swift/JournalList.swift @@ -86,7 +86,7 @@ import Observation let positions: [String: Int] init?(_ json: String) { - guard let properties = try? JSONDecoder().decode(Properties.self, from: Data(json.utf8)) + guard let properties = try? JournalInternalJSON.decode(Properties.self, from: Data(json.utf8)) else { return nil } var positions: [String: Int] = [:] var keys: Set = [] diff --git a/swift/JournalLocalAccountBindingStore.swift b/swift/JournalLocalAccountBindingStore.swift index efbb606f..d0a1e9b4 100644 --- a/swift/JournalLocalAccountBindingStore.swift +++ b/swift/JournalLocalAccountBindingStore.swift @@ -6,6 +6,7 @@ enum JournalLocalAccountBindingStore { private static let maximumBytes = 4_096 private struct Binding: Codable { let version: Int; let userId: String; let managedSyncOrigin: String } + // Version projection only; Binding below admits the complete closed record. private struct Format: Decodable { let version: Int } static func query() -> [CFString: Any] { @@ -45,7 +46,7 @@ enum JournalLocalAccountBindingStore { } let binding: Binding do { - binding = try JSONDecoder().decode(Binding.self, from: data) + binding = try JournalInternalJSON.decode(Binding.self, from: data) _ = try encode(userID: binding.userId, managedSyncOrigin: binding.managedSyncOrigin) } catch { throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead, formatVersion: version) diff --git a/swift/JournalNativeServices.swift b/swift/JournalNativeServices.swift index 543721a5..39cb652d 100644 --- a/swift/JournalNativeServices.swift +++ b/swift/JournalNativeServices.swift @@ -102,7 +102,7 @@ private final class JournalNativeAccountStorage: Sendable { } let recovery: Recovery do { - recovery = try JSONDecoder().decode(Recovery.self, from: bytes) + recovery = try JournalInternalJSON.decode(Recovery.self, from: bytes) guard recovery.version == 1 else { throw JournalPlatformServices.Failure.unavailable } } catch { throw JournalAccountIssue(kind: .unavailable, stage: .bindingWrite) diff --git a/swift/JournalPlatformWire.swift b/swift/JournalPlatformWire.swift index 12a29236..af5f0764 100644 --- a/swift/JournalPlatformWire.swift +++ b/swift/JournalPlatformWire.swift @@ -153,7 +153,7 @@ enum JournalPlatformWire { } private static func decode(_ type: Value.Type, _ payload: Data) throws -> Value { - do { return try JSONDecoder().decode(type, from: payload) } + do { return try JournalInternalJSON.decode(type, from: payload) } catch { throw Failure.invalidPacket } } diff --git a/test/dune b/test/dune index ad8e0bc4..646d29e4 100644 --- a/test/dune +++ b/test/dune @@ -124,6 +124,7 @@ (deps test_wrapped_graph_key_crypto.sh wrapped_graph_key_crypto_test.swift + ../swift/JournalInternalJSON.swift ../swift/JournalE2EECrypto.swift) (action (run sh %{dep:test_wrapped_graph_key_crypto.sh} %{workspace_root}))) diff --git a/test/source_boundary_test.ml b/test/source_boundary_test.ml index dfd2c2e4..c9d823a3 100644 --- a/test/source_boundary_test.ml +++ b/test/source_boundary_test.ml @@ -639,7 +639,8 @@ let test_standalone_sync_protocol_boundary root = [ "module Protocol = Logseq_sync_pure_reducer.Sync_protocol" ; "all client messages round trip" ; "all server messages round trip" - ; "unknown fields are strict and safe" + ; "owned client fields are strict and safe" + ; "external server extensions are open" ; "protocol validation is fail closed" ] ;; diff --git a/test/test_wrapped_graph_key_crypto.sh b/test/test_wrapped_graph_key_crypto.sh index e557eb65..6f7317bc 100644 --- a/test/test_wrapped_graph_key_crypto.sh +++ b/test/test_wrapped_graph_key_crypto.sh @@ -8,6 +8,7 @@ trap 'rm -f "$binary"' EXIT HUP INT TERM swiftc \ -swift-version 6 \ -D DEBUG \ + "$root/swift/JournalInternalJSON.swift" \ "$root/swift/JournalE2EECrypto.swift" \ "$root/test/wrapped_graph_key_crypto_test.swift" \ -o "$binary" diff --git a/tool/build_authentication_probe.py b/tool/build_authentication_probe.py index e2c83fe3..91f4e5c5 100644 --- a/tool/build_authentication_probe.py +++ b/tool/build_authentication_probe.py @@ -25,7 +25,7 @@ 'UIInterfaceOrientationLandscapeLeft', 'UIInterfaceOrientationLandscapeRight'], })) sources = [root/'swift/JournalAuthentication.swift', root/'swift/JournalAuthenticationView.swift', - root/'swift/JournalPlatformWire.swift', root/'swift/JournalPlatformServices.swift', + root/'swift/JournalInternalJSON.swift', root/'swift/JournalPlatformWire.swift', root/'swift/JournalPlatformServices.swift', root/'swift/JournalLocalAccountBindingStore.swift', root/'apple-tests/authentication/JournalAuthenticationProbe.swift'] sdk = subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', '--show-sdk-path'], text=True).strip() diff --git a/tool/test_asset_interop.py b/tool/test_asset_interop.py index bb23c5b3..29eca3c0 100644 --- a/tool/test_asset_interop.py +++ b/tool/test_asset_interop.py @@ -22,7 +22,7 @@ print(work, flush=True) record = {"commands": [], "sources": {}, "result": "incomplete"} for source in [crypt, upstream / "deps/db/src/logseq/db/sqlite/util.cljs", - root / "swift/JournalE2EECrypto.swift", + root / "swift/JournalInternalJSON.swift", root / "swift/JournalE2EECrypto.swift", root / "logseq_sync/lib/effect_runner/protocol/asset_codec.ml", root / "logseq_sync/test/asset_native_interop.ml", root / "logseq_sync/test/asset_adapter_memory.ml", @@ -67,7 +67,7 @@ def run(command, env=None): run(["swiftc", "--version"]) run(nbb + ["generate", str(work)]) library = work / "libJournalAssetCrypto.dylib" -run(["swiftc", "-emit-library", str(root / "swift/JournalE2EECrypto.swift"), +run(["swiftc", "-emit-library", str(root / "swift/JournalInternalJSON.swift"), str(root / "swift/JournalE2EECrypto.swift"), "-o", str(library)]) run(["dune", "build", "logseq_sync/test/asset_native_interop.exe", "logseq_sync/test/asset_adapter_memory.exe"]) diff --git a/tool/test_json_boundaries.py b/tool/test_json_boundaries.py new file mode 100644 index 00000000..cc7e8fc3 --- /dev/null +++ b/tool/test_json_boundaries.py @@ -0,0 +1,17 @@ +"""Public native owner and internal decoder tests; synthetic bytes, no App or credentials.""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[1] +with tempfile.TemporaryDirectory(prefix='journal-json-boundaries-') as directory: + output = Path(directory) / 'tests' + sources = ['JournalInternalJSON.swift', 'JournalPlatformWire.swift', + 'JournalPlatformServices.swift', 'JournalLocalAccountBindingStore.swift', + 'JournalCognitoOAuth.swift'] + subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', + str(Path(directory)/'modules'), + *[str(root/'swift'/name) for name in sources], + str(root/'apple-tests/platform-wire/JournalJSONBoundaryTests.swift'), + '-o',str(output)],check=True) + subprocess.run([str(output)],check=True,timeout=60) diff --git a/tool/test_swiftui_cognito.py b/tool/test_swiftui_cognito.py index 9ff9abd7..9182bafd 100644 --- a/tool/test_swiftui_cognito.py +++ b/tool/test_swiftui_cognito.py @@ -5,7 +5,7 @@ root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix='journal-cognito-') as directory: destination = Path(directory) - sources = ['JournalPlatformWire.swift', 'JournalPlatformServices.swift', + sources = ['JournalInternalJSON.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', 'JournalAuthentication.swift', 'JournalCognitoOAuth.swift', 'JournalCognitoSession.swift'] subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-Xfrontend', '-disable-sandbox', '-module-cache-path', str(destination/'modules'), *[str(root/'swift'/s) for s in sources], diff --git a/tool/test_swiftui_events.py b/tool/test_swiftui_events.py index 4a09eccf..c8fcff2e 100644 --- a/tool/test_swiftui_events.py +++ b/tool/test_swiftui_events.py @@ -6,8 +6,8 @@ root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix='journal-platform-events-') as directory: executable = Path(directory) / 'events-tests' - subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', - str(root / 'swift/JournalPlatformWire.swift'), + subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), + str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), str(root / 'swift/JournalPlatformEvents.swift'), str(root / 'apple-tests/platform-events/JournalPlatformEventsTests.swift'), '-o', str(executable)], check=True) diff --git a/tool/test_swiftui_platform.py b/tool/test_swiftui_platform.py index 052aaebb..f866508b 100644 --- a/tool/test_swiftui_platform.py +++ b/tool/test_swiftui_platform.py @@ -58,7 +58,7 @@ def run(directory, command): *[source.stem + '.cmx' for source in sources if source.suffix == '.ml']]) run(destination, [str(destination / 'wire_ocaml'), 'emit', 'requests.json']) run(destination, ['swiftc', '-swift-version', '6', '-module-cache-path', str(destination / 'module-cache'), '-o', 'wire_swift', - str(ROOT / 'swift/JournalPlatformWire.swift'), + str(ROOT / 'swift/JournalInternalJSON.swift'), str(ROOT / 'swift/JournalPlatformWire.swift'), str(ROOT / 'swift/JournalPlatformServices.swift'), str(ROOT / 'swift/JournalStartupConfiguration.swift'), str(ROOT / 'swift/JournalLocalAccountBindingStore.swift'), diff --git a/tool/test_swiftui_services.py b/tool/test_swiftui_services.py index 88822e0f..b097aaaa 100644 --- a/tool/test_swiftui_services.py +++ b/tool/test_swiftui_services.py @@ -7,7 +7,7 @@ with tempfile.TemporaryDirectory(prefix='journal-platform-services-') as directory: executable = Path(directory)/'services-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), - str(root/'swift/JournalPlatformWire.swift'), + str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'apple-tests/platform-services/JournalPlatformServicesTests.swift'), '-o', str(executable)], check=True) @@ -15,7 +15,7 @@ recovery = Path(directory)/'recovery-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-Xfrontend', '-disable-sandbox', '-module-cache-path', str(Path(directory)/'modules'), - str(root/'swift/JournalPlatformWire.swift'), + str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalLocalAccountBindingStore.swift'), str(root/'apple-tests/platform-services/JournalAccountRecoveryTests.swift'), @@ -23,7 +23,7 @@ subprocess.run([str(recovery)], check=True, timeout=60) threading = Path(directory)/'account-threading-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), - str(root/'swift/JournalPlatformWire.swift'), + str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalStartupConfiguration.swift'), str(root/'swift/JournalNativeServices.swift'), @@ -32,7 +32,7 @@ subprocess.run([str(threading)], check=True, timeout=60) cold = Path(directory)/'cold-tests' subprocess.run(['xcrun','swiftc','-swift-version','6','-module-cache-path',str(Path(directory)/'modules'), - str(root/'swift/JournalPlatformWire.swift'), + str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalStartupConfiguration.swift'), str(root/'swift/JournalNativeServices.swift'), diff --git a/tool/test_swiftui_warm_start.py b/tool/test_swiftui_warm_start.py index 9266ab0c..06b32d75 100644 --- a/tool/test_swiftui_warm_start.py +++ b/tool/test_swiftui_warm_start.py @@ -97,7 +97,7 @@ def run(command, cwd=host): generator.write_text(bootstrap+'\n#load '+json.dumps(str(fixture_source.with_suffix('.cmo')))+';;\n') sources = [root/'swift'/name for name in - ['JournalE2EECrypto.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', + ['JournalInternalJSON.swift', 'JournalE2EECrypto.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', 'JournalStartupConfiguration.swift', 'JournalChrome.swift']] source = root/'apple-tests/warm-start/JournalWarmStartAcceptance.swift' for tracked in sources + [source]: From 79c2b0ad98aee1a559e2868c57bb2c303313b38c Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 14 Oct 2026 15:57:35 +0800 Subject: [PATCH 2/4] Verify the deployed crypto provider compiles without authentication --- .github/workflows/ci.yml | 2 + .../2026-10-14-external-input-open-json.md | 2 + .../test/test_managed_sync_e2e.ml | 57 +++++++++++-------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 778f7ac8..1cb1844b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: - name: Build all targets and native embed run: | opam exec -- dune build @all app/native_embed.exe.o + - name: Verify deployed crypto provider compiles offline + run: opam exec -- dune exec logseq_db_worker/test/test_managed_sync_e2e.exe -- --compile-platform-crypto-only - name: Run full regression suite run: opam exec -- dune runtest --force - name: Upload resolved dependencies and test logs diff --git a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md index 0f67635f..84125233 100644 --- a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md @@ -66,3 +66,5 @@ Current upstream dev `949a4f0e72cddb2dad34091393bfa09592171944` declares optiona The complete build (`@all` and native embed), full regression recheck (259 sync cases), 10 native JSON boundary assertions, account/recovery/event and synthetic signed-Cognito suites, public OCaml-to-Swift wire round trip and production iOS SwiftUI typecheck passed. The initial concurrent full run exposed one existing 20ms/30ms RFC Pong timing failure; both original and restored protocol single-case checks then passed, followed by the full regression recheck. No transport timing or production transport code was changed. The document passes spec-dev-tool; check --all reports three unchanged preexisting invalid historical decisions. A preliminary independent read-only review of the uncommitted diff using journal-code-review found no evidenced P1/P2. It does not replace the independent Codex session review of the formal PR snapshot. + +The formal independent Codex review of head 3511e42 found one P2: the deployed E2E provider compiler still passed only JournalE2EECrypto.swift. Its actual compiler function now includes JournalInternalJSON.swift. A compile-only public executable mode stops before provider loading, environment/credential reading, Keychain creation or networking; this mode reproduced the missing-type compiler error (exit 2) and passed after the source-list fix (exit 0). CI runs this offline compiler regression after building targets. No online E2E was executed. diff --git a/logseq_db_worker/test/test_managed_sync_e2e.ml b/logseq_db_worker/test/test_managed_sync_e2e.ml index 2adf35b1..f63ee36c 100644 --- a/logseq_db_worker/test/test_managed_sync_e2e.ml +++ b/logseq_db_worker/test/test_managed_sync_e2e.ml @@ -1141,7 +1141,8 @@ let process_exit_code = function let compile_platform_crypto_provider ~root ~output = let compiler = "/usr/bin/xcrun" in let source = Filename.concat root "swift/JournalE2EECrypto.swift" in - if not (Sys.file_exists source) + let decoder = Filename.concat root "swift/JournalInternalJSON.swift" in + if not (Sys.file_exists source && Sys.file_exists decoder) then fail "the production platform crypto provider source is unavailable"; let arguments = [| compiler @@ -1150,6 +1151,7 @@ let compile_platform_crypto_provider ~root ~output = ; "DEBUG" ; "-parse-as-library" ; "-emit-library" + ; decoder ; source ; "-o" ; output @@ -1163,7 +1165,7 @@ let compile_platform_crypto_provider ~root ~output = then fail "the production platform crypto provider failed to compile" ;; -let run_with_platform_crypto_provider () = +let run_with_platform_crypto_provider ?(compile_only = false) () = let root = match Sys.getenv_opt "DUNE_SOURCEROOT" with | Some root when Sys.file_exists (Filename.concat root "dune-project") -> root @@ -1187,27 +1189,32 @@ let run_with_platform_crypto_provider () = Unix.rmdir directory)) (fun () -> compile_platform_crypto_provider ~root ~output:provider; - let environment = - environment_with (Unix.environment ()) "DYLD_INSERT_LIBRARIES" provider - |> fun environment -> - environment_with environment platform_crypto_provider_marker "1" - |> fun environment -> - environment_with environment test_file_keychain_environment keychain - in - let executable = Unix.realpath Sys.executable_name in - let arguments = Array.copy Sys.argv in - arguments.(0) <- executable; - let process = - Unix.create_process_env - executable - arguments - environment - Unix.stdin - Unix.stdout - Unix.stderr - in - let _, status = Unix.waitpid [] process in - process_exit_code status) + (* Offline compiler verification never loads the provider, reads credentials, + creates a Keychain or starts a network client. *) + if compile_only + then 0 + else ( + let environment = + environment_with (Unix.environment ()) "DYLD_INSERT_LIBRARIES" provider + |> fun environment -> + environment_with environment platform_crypto_provider_marker "1" + |> fun environment -> + environment_with environment test_file_keychain_environment keychain + in + let executable = Unix.realpath Sys.executable_name in + let arguments = Array.copy Sys.argv in + arguments.(0) <- executable; + let process = + Unix.create_process_env + executable + arguments + environment + Unix.stdin + Unix.stdout + Unix.stderr + in + let _, status = Unix.waitpid [] process in + process_exit_code status)) ;; let with_execution_lock f = @@ -1351,7 +1358,9 @@ let run () = let () = let status = try - if Sys.getenv_opt platform_crypto_provider_marker = Some "1" + if Array.length Sys.argv = 2 && Sys.argv.(1) = "--compile-platform-crypto-only" + then run_with_platform_crypto_provider ~compile_only:true () + else if Sys.getenv_opt platform_crypto_provider_marker = Some "1" then ( with_execution_lock run; 0) From 9293badbbc05775016781a9b31702bbc9211bcf3 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 14 Oct 2026 16:24:27 +0800 Subject: [PATCH 3/4] Limit JSON codec changes to OCaml and restore Swift behavior --- .github/workflows/ci.yml | 4 - .../JournalJSONBoundaryTests.swift | 90 ----------- .../2026-10-14-external-input-open-json.md | 65 +++----- .../test/test_managed_sync_e2e.ml | 57 +++---- swift/JournalCognitoNative.swift | 2 +- swift/JournalE2EECrypto.swift | 2 +- swift/JournalExtensions.swift | 2 +- swift/JournalInternalJSON.swift | 146 ------------------ swift/JournalList.swift | 2 +- swift/JournalLocalAccountBindingStore.swift | 3 +- swift/JournalNativeServices.swift | 2 +- swift/JournalPlatformWire.swift | 2 +- test/dune | 1 - test/test_wrapped_graph_key_crypto.sh | 1 - tool/build_authentication_probe.py | 2 +- tool/test_asset_interop.py | 4 +- tool/test_json_boundaries.py | 17 -- tool/test_swiftui_cognito.py | 2 +- tool/test_swiftui_events.py | 4 +- tool/test_swiftui_platform.py | 2 +- tool/test_swiftui_services.py | 8 +- tool/test_swiftui_warm_start.py | 2 +- 22 files changed, 63 insertions(+), 357 deletions(-) delete mode 100644 apple-tests/platform-wire/JournalJSONBoundaryTests.swift delete mode 100644 swift/JournalInternalJSON.swift delete mode 100644 tool/test_json_boundaries.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cb1844b..9c4846b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,8 +17,6 @@ jobs: - uses: actions/checkout@v4 - name: Verify Apple app icon packaging run: python3 tool/test_apple_app_icon.py - - name: Verify owned and external native JSON boundaries - run: python3 tool/test_json_boundaries.py - uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: "5.5.0" @@ -39,8 +37,6 @@ jobs: - name: Build all targets and native embed run: | opam exec -- dune build @all app/native_embed.exe.o - - name: Verify deployed crypto provider compiles offline - run: opam exec -- dune exec logseq_db_worker/test/test_managed_sync_e2e.exe -- --compile-platform-crypto-only - name: Run full regression suite run: opam exec -- dune runtest --force - name: Upload resolved dependencies and test logs diff --git a/apple-tests/platform-wire/JournalJSONBoundaryTests.swift b/apple-tests/platform-wire/JournalJSONBoundaryTests.swift deleted file mode 100644 index ff7bc8a6..00000000 --- a/apple-tests/platform-wire/JournalJSONBoundaryTests.swift +++ /dev/null @@ -1,90 +0,0 @@ -import Foundation - -@main @MainActor struct JournalJSONBoundaryTests { - private struct Child: Decodable { let name: String; let note: String? } - private struct Internal: Decodable { let children: [Child]; let named: [String: Child] } - private enum Choice: Decodable { case value(Child) } - private struct FoundationValues: Codable { - let data: Data; let date: Date; let url: URL; let decimal: Decimal - } - private static var failures = 0 - - private static func check(_ name: String, _ action: () throws -> Bool) { - do { - if try action() { print("PASS \(name)") } - else { failures += 1; print("FAIL \(name)") } - } catch { failures += 1; print("FAIL \(name): \(type(of: error))") } - } - private static func rejected(_ action: () throws -> Void) -> Bool { - do { try action(); return false } catch { return true } - } - private static func packet(_ payload: String) -> Data { - var bytes = Data([76,74,80,50,2,0,8,0] + Array(repeating: UInt8(0), count:24)) - let data = Data(payload.utf8) - for n in 0..<4 { bytes[24+n] = UInt8((data.count >> (8*n)) & 255) } - bytes.append(data) - return bytes - } - - static func main() throws { - check("public native wire rejects extra internal fields") { - rejected { _ = try JournalPlatformWire.decodeRequest(packet(#"{"challengeId":"synthetic","future":true}"#)) } - } - check("public native wire retains required and type validation") { - [#"{"future":true}"#, #"{"challengeId":null}"#, #"{"challengeId":1}"#].allSatisfy { - input in rejected { _ = try JournalPlatformWire.decodeRequest(packet(input)) } - } - } - check("public account binding rejects extra internal fields") { - rejected { _ = try JournalLocalAccountBindingStore.decode(Data(#"{"version":2,"userId":"synthetic","managedSyncOrigin":"https://example.invalid","future":true}"#.utf8)) } - } - check("recursive internal decoder rejects root nested array and dictionary extensions") { - [#"{"children":[],"named":{},"future":true}"#, - #"{"children":[{"name":"synthetic","future":true}],"named":{}}"#, - #"{"children":[],"named":{"dynamic-key":{"name":"synthetic","future":true}}}"#].allSatisfy { - input in rejected { _ = try JournalInternalJSON.decode(Internal.self, from: Data(input.utf8)) } - } - } - check("closed native decoder preserves option null and dynamic dictionaries") { - let value = try JournalInternalJSON.decode(Internal.self, from: - Data(#"{"children":[{"name":"synthetic","note":null}],"named":{"dynamic-key":{"name":"synthetic"}}}"#.utf8)) - return value.children.first?.note == nil && value.named["dynamic-key"]?.name == "synthetic" - } - check("closed native decoder preserves required and scalar type constraints") { - [#"{"children":[],"named":{"dynamic-key":{}}}"#, - #"{"children":[{"name":null}],"named":{}}"#, - #"{"children":[{"name":1}],"named":{}}"#].allSatisfy { - input in rejected { _ = try JournalInternalJSON.decode(Internal.self, from: Data(input.utf8)) } - } - } - check("closed enum containers reject extra cases") { - rejected { _ = try JournalInternalJSON.decode(Choice.self, from: - Data(#"{"value":{"_0":{"name":"synthetic"}},"future":{}}"#.utf8)) } - } - check("closed field errors do not emit unknown field names or values") { - do { - _ = try JournalInternalJSON.decode(Internal.self, from: - Data(#"{"children":[],"named":{},"CANARY_FIELD_NAME":"CANARY_PRIVATE_VALUE"}"#.utf8)) - return false - } catch { return !String(describing:error).contains("CANARY") } - } - check("external Cognito response keeps open semantics") { - let response = try JSONDecoder().decode(JournalCognitoTokenResponse.self, from: - Data(#"{"id_token":"synthetic","access_token":"synthetic","expires_in":3600,"token_type":"Bearer","future":{"nested":true}}"#.utf8)) - return response.expires_in == 3600 && response.refresh_token == nil - } - check("closed decoder preserves Foundation scalar roots and nested values") { - let encoder = JSONEncoder() - let bytes = Data([1, 2, 3]) - let root = try JournalInternalJSON.decode(Data.self, from: encoder.encode(bytes)) - let number = try JournalInternalJSON.decode(Decimal.self, from: encoder.encode(Decimal(12))) - let value = FoundationValues(data: bytes, date: Date(timeIntervalSinceReferenceDate: 1), - url: URL(string: "https://example.invalid")!, decimal: Decimal(12)) - let decoded = try JournalInternalJSON.decode(FoundationValues.self, from: encoder.encode(value)) - let array = try JournalInternalJSON.decode([Data].self, from: encoder.encode([bytes])) - return root == bytes && number == 12 && decoded.data == bytes && decoded.date == value.date - && decoded.url == value.url && decoded.decimal == value.decimal && array == [bytes] - } - if failures > 0 { exit(1) } - } -} diff --git a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md index 84125233..13c04a33 100644 --- a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md @@ -1,70 +1,45 @@ -# Open external JSON inputs and close owned JSON contracts +# Open external OCaml JSON inputs and keep owned contracts closed ## Problem -A real Journal sync error rejects the extra tx-id field on pull/ok.txs[0] before any transaction is applied. The generated server DTO and manual structural whitelist both reject it. External services can extend responses without changing the fields Journal consumes. Shared input/output DTOs also obscure which side owns a schema; synthesized Swift internal decoders currently tolerate unknown fields. +A real Journal sync error rejects pull/ok.txs[0].tx-id before transactions are applied. Both the generated server DTO and the structural whitelist reject external response extensions. External services can add fields Journal does not consume without changing required fields. ## Decision -Follow the user-approved boundary policy. External server/HTTP JSON input DTOs derive open PPX readers, including nested records; validate declared required fields, their types and domain semantics. Continue rejecting unsupported message variants and malformed JSON. Ignore unused pull tx-id regardless of its value. Keep generated closed output DTOs separate, with semantic and wire budget checks; client wire validation remains closed. Internal OCaml commands, outcomes and persistence domain codecs remain closed. Swift-owned internal/native receivers use a recursive Decoder wrapper that rejects keys absent from synthesized CodingKeys; external Cognito and Transit readers do not use it. Keep generated Codable/PPX construction, existing keys and formats. +Limit this PR to OCaml JSON codecs, as explicitly requested by the user. Keep PPX. External WS and E2EE HTTP records use separate input-only open readers, including nested records; declared required fields, types, nullability, supported variants, semantics and size budgets remain validated. Unused pull tx-id is ignored. Owned client/output DTOs remain closed, and existing internal Config/Protocol/full persistence admission stays closed. Catalog and snapshot metadata retain open input behavior with decode-only derivations. Outbox metadata readers remain decode-only projections rather than full-record admission. -The existing metadata-only outbox/format readers are projections of a complete record, not full-record admission or constructors; make decode-only derivations explicit where applicable. Their extra declared owner fields must not be rejected as projection extras. The domain owner remains responsible for the full closed contract. +All Swift codec behavior and its compiler/test/CI configuration are restored exactly to the PR's merge-base. The generic Swift decoder, native call-site replacements, native boundary tests and offline provider compile mode are outside the narrowed scope. ## Alternatives considered ### Declare only tx-id -Rejected because the user explicitly authorized open external inputs, including nested future fields. Adding a whitelist entry would leave the same failure for the next external extension. +Would still reject the next external extension. The user requested open external input schemas. -### Make all shared DTOs open +### Make shared input/output DTOs open -Rejected because it would extend open semantics to internal receivers and weaken validation of owned output. Use separate external-input and closed-output DTOs. +Would spread external tolerance into owned contracts. Keep separate input-only readers and closed output DTOs. -## Acceptance criteria +### Add strict Swift receiving infrastructure -- Public Sync_protocol API accepts top-level and nested server extensions while preserving required/type/null/semantic, cursor/budget and discriminator failures; all client input remains closed and generated outbound shapes contain only contract fields. -- Public Runner requests exercise open HTTP catalog/E2EE inputs through the existing isolated TLS peer. Core public completions retain snapshot metadata field/URL validation. -- Native public wire and closed-decoder utility tests reject root/nested/array extras, retain required/type/null/default behavior and leave external Cognito decode open. -- Tests are written before implementation and demonstrably RED for the old behavior, then GREEN. Existing targeted regressions and appropriate broader checks pass. -- No Dune/spec files, UI PR73 checkout, user graph, credentials, phone installation or temporary diagnostic observer is changed. -- Create a formal PR, freeze head SHA, wait for that exact head CI terminal results, then request independent journal-code-review. Do not merge. - -## Risks - -- Open schemas still enforce standard JSON safety and declared semantic constraints; unknown event types are not extension fields. -- Swift native receiver fields must match the complete OCaml producer contract. Nested checking must preserve optional nulls, arrays and dynamic dictionaries rather than guess field lists. -- Public Core receives already-decoded WS messages and HTTP key values, so it cannot reproduce raw field rejection. The narrowest raw WS boundary is public Sync_protocol; HTTP raw parsing belongs to the public Runner and its transport peer. Do not inject an already incorrect parsed result into a reducer as a reproduction. -- Existing live evidence covers the reported WS extra-field rejection and historical catalog/E2EE paths, plus a public JWKS read. Synthetic tests are not live service validation; actual signed-in response variants remain unobserved. - -## Questions - -- None. The user expressly authorized this policy change and a separate formal PR; any necessary Dune/spec edit would require a separate minimal-diff request. +Not part of the narrowed OCaml scope. Generated Encodable/PPX writers already emit their declared fields; strict receiving is a separate defensive policy. Restore Swift and dependent wiring to their prior behavior. ## Consequences -WS server records now have separate input-only open PPX DTOs. The discriminator remains owned by its header reader and semantic validator. Open object validation checks declared values and duplicate safety without running the owned-field whitelist. Output uses the original closed DTO serializers and generated closed readers, plus existing semantic and size validation. HTTP E2EE inputs are open; catalog and snapshot metadata retain their already-open behavior but expose no encoders. Internal Config/Protocol/full persistence codecs stay closed. Decode-only outbox indexing and account-format projections remain explicitly separate from full-record admission. +Open object validation checks declared fields and duplicate safety without applying the owned-field whitelist. Output uses the original generated closed DTOs and readers with semantic/budget validation. Internal full-record codecs stay closed; projections do not expose encoders. All Dune/spec/Swift/CI files have no net changes relative to the merge-base. No UI changes, real account/graph writes, new diagnostics, merge or phone installation belong to this PR. -Swift native wire, extension properties, persisted token/binding/recovery records and the crypto bridge use the closed recursive Decoder. It derives allowed keys from each Decodable container's synthesized CodingKeys, including nested arrays/enums; dynamic dictionaries keep their intentional key space. Foundation's Data/Date/URL/Decimal decoding is preserved. External OAuth/JWT/JWKS/Transit readers keep their existing open behavior. Unknown-field diagnostics omit unknown names and values. +## Validation and coverage -The user explicitly approved the one-line JournalInternalJSON.swift dependency in the existing wrapped-key crypto Dune rule. It is the only Dune change; all OCaml spec files remain untouched. +Tests were written first. Against unchanged main953, public WS extension/declared-control tests failed, and public Runner/local TLS HTTP tests failed four extension-acceptance assertions; required/type/null/duplicate controls passed. Core metadata and internal Config controls passed at baseline. These are synthetic regression payloads, not live service compatibility claims. -## Validation and actual-data coverage - -Tests were added first. With production at main953, the public WS codec failed its two new extension/declared-constraint tests, native wire/binding plus recursive utility behavior failed five extension assertions, and the public Runner/local TLS peer failed four E2EE extension-acceptance assertions. Existing required/type/null/duplicate controls passed. The Core metadata and internal Config controls passed at baseline. These fixtures are synthetic, including the independent local TLS peer and signed Cognito issuer. - -| Boundary | Real evidence | Regression coverage | +| Boundary | Actual evidence | Public regression boundary | | --- | --- | --- | -| WS server/client | Materialized and viewed error screenshot proves pull/ok.txs[0].tx-id rejection; full payload and deployed SHA unknown | All supported server variants/rejection reasons, top/nested extensions, ignored tx-id shapes, required/type/null/semantic/variant errors, closed client/output and wire limits | -| Catalog and E2EE HTTP | Earlier authorized App type-only logs: catalog/user keys HTTP 200; graph key before/after PR72; no new signed-in response collection | Public Runner with independent localhost TLS peer, valid extensions and invalid consumed fields | -| Cognito OAuth/JWT/JWKS | Public JWKS read returned 200, two RSA keys and decoded through the existing Swift reader; no tokens recorded | Existing production owner with synthetic signed issuer; open response and authentication failure controls | -| Snapshot metadata/baseline | Upstream schema only; no live signed-in response | Public Core raw completions verify extension tolerance, required/type/null and HTTPS URL constraints | -| Transit sync/snapshot/assets | No new live graph/asset samples | Existing envelope/size/transaction/snapshot/asset regression suites; no format changes | -| Owned native/persistence/output | OCaml producer emitted real generated bytes to the Swift public receiver in an isolated harness; no real account store accessed | Native wire round trip, nested closed decoder, optional/dictionary behavior and account recovery; existing OCaml closed persistence tests | - -Current upstream dev `949a4f0e72cddb2dad34091393bfa09592171944` declares optional non-null UUID tx-id on server tx-log entries; old `015cc9f` does not declare that server field. Both use open Malli maps. This is source evidence, not the screenshot field's observed type or deployed revision: [frozen dev schema](https://github.com/logseq/logseq/blob/949a4f0e72cddb2dad34091393bfa09592171944/deps/db-sync/src/logseq/db_sync/malli_schema.cljs#L10). Journal does not consume tx-id, so its external input reader ignores it. - -The complete build (`@all` and native embed), full regression recheck (259 sync cases), 10 native JSON boundary assertions, account/recovery/event and synthetic signed-Cognito suites, public OCaml-to-Swift wire round trip and production iOS SwiftUI typecheck passed. The initial concurrent full run exposed one existing 20ms/30ms RFC Pong timing failure; both original and restored protocol single-case checks then passed, followed by the full regression recheck. No transport timing or production transport code was changed. The document passes spec-dev-tool; check --all reports three unchanged preexisting invalid historical decisions. +| WS server/client | Materialized and viewed screenshot establishes pull/ok.txs[0].tx-id rejection; payload and deployed SHA unknown | Sync_protocol: all supported variants and rejection reasons, root/nested extras, ignored tx-id shapes, required/type/null/semantic/variant failures, closed client/output and limits | +| Catalog/E2EE HTTP | Prior authorized type-only App logs show catalog/user-key HTTP 200 and graph-key behavior before/after PR72; no fresh signed-in payload collection | Runner with independent localhost TLS peer and generated readers | +| Snapshot metadata/baseline | Upstream schema, no live signed-in response | Core raw completions preserve required/type/null/HTTPS URL checks | +| Internal/outbound | No real account record accessed | Config rejects root/nested extras; existing Protocol and persistence suites retain closed full admission; generated client/server output round trips | +| Transit/snapshot/assets | No new live graph/asset samples | Existing envelope/size/transaction/snapshot/asset regressions; formats unchanged | -A preliminary independent read-only review of the uncommitted diff using journal-code-review found no evidenced P1/P2. It does not replace the independent Codex session review of the formal PR snapshot. +Frozen upstream dev 949a4f0e72cddb2dad34091393bfa09592171944 declares optional non-null UUID tx-id in server entries; old 015cc9f does not declare that server field. Both use open Malli maps. This source evidence does not establish the screenshot value/type or deployment revision: [frozen schema](https://github.com/logseq/logseq/blob/949a4f0e72cddb2dad34091393bfa09592171944/deps/db-sync/src/logseq/db_sync/malli_schema.cljs#L10). Journal does not consume tx-id. -The formal independent Codex review of head 3511e42 found one P2: the deployed E2E provider compiler still passed only JournalE2EECrypto.swift. Its actual compiler function now includes JournalInternalJSON.swift. A compile-only public executable mode stops before provider loading, environment/credential reading, Keychain creation or networking; this mode reproduced the missing-type compiler error (exit 2) and passed after the source-list fix (exit 0). CI runs this offline compiler regression after building targets. No online E2E was executed. +After scope removal, rerun the OCaml build/native embed and regression suite, review the exact new head in the existing independent Codex session using journal-code-review, and track its CI to success. Prior Swift-inclusive review/CI snapshots are superseded. spec-dev-tool check --all has three preexisting invalid historical decisions unrelated to this change. diff --git a/logseq_db_worker/test/test_managed_sync_e2e.ml b/logseq_db_worker/test/test_managed_sync_e2e.ml index f63ee36c..2adf35b1 100644 --- a/logseq_db_worker/test/test_managed_sync_e2e.ml +++ b/logseq_db_worker/test/test_managed_sync_e2e.ml @@ -1141,8 +1141,7 @@ let process_exit_code = function let compile_platform_crypto_provider ~root ~output = let compiler = "/usr/bin/xcrun" in let source = Filename.concat root "swift/JournalE2EECrypto.swift" in - let decoder = Filename.concat root "swift/JournalInternalJSON.swift" in - if not (Sys.file_exists source && Sys.file_exists decoder) + if not (Sys.file_exists source) then fail "the production platform crypto provider source is unavailable"; let arguments = [| compiler @@ -1151,7 +1150,6 @@ let compile_platform_crypto_provider ~root ~output = ; "DEBUG" ; "-parse-as-library" ; "-emit-library" - ; decoder ; source ; "-o" ; output @@ -1165,7 +1163,7 @@ let compile_platform_crypto_provider ~root ~output = then fail "the production platform crypto provider failed to compile" ;; -let run_with_platform_crypto_provider ?(compile_only = false) () = +let run_with_platform_crypto_provider () = let root = match Sys.getenv_opt "DUNE_SOURCEROOT" with | Some root when Sys.file_exists (Filename.concat root "dune-project") -> root @@ -1189,32 +1187,27 @@ let run_with_platform_crypto_provider ?(compile_only = false) () = Unix.rmdir directory)) (fun () -> compile_platform_crypto_provider ~root ~output:provider; - (* Offline compiler verification never loads the provider, reads credentials, - creates a Keychain or starts a network client. *) - if compile_only - then 0 - else ( - let environment = - environment_with (Unix.environment ()) "DYLD_INSERT_LIBRARIES" provider - |> fun environment -> - environment_with environment platform_crypto_provider_marker "1" - |> fun environment -> - environment_with environment test_file_keychain_environment keychain - in - let executable = Unix.realpath Sys.executable_name in - let arguments = Array.copy Sys.argv in - arguments.(0) <- executable; - let process = - Unix.create_process_env - executable - arguments - environment - Unix.stdin - Unix.stdout - Unix.stderr - in - let _, status = Unix.waitpid [] process in - process_exit_code status)) + let environment = + environment_with (Unix.environment ()) "DYLD_INSERT_LIBRARIES" provider + |> fun environment -> + environment_with environment platform_crypto_provider_marker "1" + |> fun environment -> + environment_with environment test_file_keychain_environment keychain + in + let executable = Unix.realpath Sys.executable_name in + let arguments = Array.copy Sys.argv in + arguments.(0) <- executable; + let process = + Unix.create_process_env + executable + arguments + environment + Unix.stdin + Unix.stdout + Unix.stderr + in + let _, status = Unix.waitpid [] process in + process_exit_code status) ;; let with_execution_lock f = @@ -1358,9 +1351,7 @@ let run () = let () = let status = try - if Array.length Sys.argv = 2 && Sys.argv.(1) = "--compile-platform-crypto-only" - then run_with_platform_crypto_provider ~compile_only:true () - else if Sys.getenv_opt platform_crypto_provider_marker = Some "1" + if Sys.getenv_opt platform_crypto_provider_marker = Some "1" then ( with_execution_lock run; 0) diff --git a/swift/JournalCognitoNative.swift b/swift/JournalCognitoNative.swift index 4c9e7ca5..3a20c806 100644 --- a/swift/JournalCognitoNative.swift +++ b/swift/JournalCognitoNative.swift @@ -37,7 +37,7 @@ private final class JournalCognitoKeychain: Sendable { if status == errSecItemNotFound { return nil } guard status == errSecSuccess, let data = item as? Data, data.count <= 131072 else { throw JournalCognitoError.storage } - return try JournalInternalJSON.decode(JournalCognitoTokens.self, from: data) + return try JSONDecoder().decode(JournalCognitoTokens.self, from: data) } } func save(_ tokens: JournalCognitoTokens) async throws { diff --git a/swift/JournalE2EECrypto.swift b/swift/JournalE2EECrypto.swift index fe06b5aa..65b4da1e 100644 --- a/swift/JournalE2EECrypto.swift +++ b/swift/JournalE2EECrypto.swift @@ -1119,7 +1119,7 @@ public func logseqJournalCryptoJSON( let requestPointer, let data = String(cString: requestPointer).data(using: .utf8) else { throw JournalE2EECryptoError.invalidRequest } - let request = try JournalInternalJSON.decode(JournalCryptoRequest.self, from: data) + let request = try JSONDecoder().decode(JournalCryptoRequest.self, from: data) if ProcessInfo.processInfo.environment["JOURNAL_UNLOCK_DIAGNOSTICS"] == "1" { trace = JournalUnlockTrace(request) JournalUnlockTrace.current = trace diff --git a/swift/JournalExtensions.swift b/swift/JournalExtensions.swift index 9cae8fa6..abd3a4c8 100644 --- a/swift/JournalExtensions.swift +++ b/swift/JournalExtensions.swift @@ -96,7 +96,7 @@ enum JournalExtensionFingerprint { _ type: Properties.Type, context: LUIAppleExtensionViewContext ) -> Properties? { guard case let .string(json) = context.property("payload"), - let value = try? JournalInternalJSON.decode(type, from: Data(json.utf8)) + let value = try? JSONDecoder().decode(type, from: Data(json.utf8)) else { return nil } return value } diff --git a/swift/JournalInternalJSON.swift b/swift/JournalInternalJSON.swift deleted file mode 100644 index 7d36edca..00000000 --- a/swift/JournalInternalJSON.swift +++ /dev/null @@ -1,146 +0,0 @@ -import Foundation - -/// Owned JSON uses synthesized CodingKeys as its closed schema, recursively. -/// External service readers continue using their ordinary JSONDecoder. -enum JournalInternalJSON { - static func decode(_ type: Value.Type, from data: Data) throws -> Value { - let decoder = JSONDecoder() - if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { - return try decoder.decode(type, from: data) - } - return try decoder.decode(OwnedValue.self, from: data).value - } -} - -private struct AnyField: CodingKey { - let stringValue: String - let intValue: Int? - init?(stringValue: String) { self.stringValue = stringValue; intValue = nil } - init?(intValue: Int) { stringValue = String(intValue); self.intValue = intValue } -} - -private struct OwnedValue: Decodable { - let value: Value - init(from decoder: Decoder) throws { value = try Value(from: OwnedDecoder(base: decoder)) } -} - -private struct OwnedDecoder: Decoder { - let base: Decoder - var codingPath: [CodingKey] { base.codingPath } - var userInfo: [CodingUserInfoKey: Any] { base.userInfo } - - func container(keyedBy type: Key.Type) throws -> KeyedDecodingContainer { - let fields = try base.container(keyedBy: AnyField.self).allKeys - guard fields.allSatisfy({ Key(stringValue: $0.stringValue) != nil }) else { - throw DecodingError.dataCorrupted(.init(codingPath: codingPath, - debugDescription: "Unexpected field in owned JSON object")) - } - return KeyedDecodingContainer(OwnedKeyed(base: try base.container(keyedBy: type))) - } - func unkeyedContainer() throws -> UnkeyedDecodingContainer { - OwnedUnkeyed(base: try base.unkeyedContainer()) - } - func singleValueContainer() throws -> SingleValueDecodingContainer { - OwnedSingle(base: try base.singleValueContainer(), decoder: self) - } -} - -private struct OwnedKeyed: KeyedDecodingContainerProtocol { - let base: KeyedDecodingContainer - var codingPath: [CodingKey] { base.codingPath } - var allKeys: [Key] { base.allKeys } - func contains(_ key: Key) -> Bool { base.contains(key) } - func decodeNil(forKey key: Key) throws -> Bool { try base.decodeNil(forKey: key) } - func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool { try base.decode(type, forKey: key) } - func decode(_ type: String.Type, forKey key: Key) throws -> String { try base.decode(type, forKey: key) } - func decode(_ type: Double.Type, forKey key: Key) throws -> Double { try base.decode(type, forKey: key) } - func decode(_ type: Float.Type, forKey key: Key) throws -> Float { try base.decode(type, forKey: key) } - func decode(_ type: Int.Type, forKey key: Key) throws -> Int { try base.decode(type, forKey: key) } - func decode(_ type: Int8.Type, forKey key: Key) throws -> Int8 { try base.decode(type, forKey: key) } - func decode(_ type: Int16.Type, forKey key: Key) throws -> Int16 { try base.decode(type, forKey: key) } - func decode(_ type: Int32.Type, forKey key: Key) throws -> Int32 { try base.decode(type, forKey: key) } - func decode(_ type: Int64.Type, forKey key: Key) throws -> Int64 { try base.decode(type, forKey: key) } - func decode(_ type: UInt.Type, forKey key: Key) throws -> UInt { try base.decode(type, forKey: key) } - func decode(_ type: UInt8.Type, forKey key: Key) throws -> UInt8 { try base.decode(type, forKey: key) } - func decode(_ type: UInt16.Type, forKey key: Key) throws -> UInt16 { try base.decode(type, forKey: key) } - func decode(_ type: UInt32.Type, forKey key: Key) throws -> UInt32 { try base.decode(type, forKey: key) } - func decode(_ type: UInt64.Type, forKey key: Key) throws -> UInt64 { try base.decode(type, forKey: key) } - func decode(_ type: Value.Type, forKey key: Key) throws -> Value { - // Preserve Foundation's specialized value decoding (not record schemas). - if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { - return try base.decode(type, forKey: key) - } - return try base.decode(OwnedValue.self, forKey: key).value - } - func nestedContainer(keyedBy type: Nested.Type, forKey key: Key) throws -> KeyedDecodingContainer { - try OwnedDecoder(base: base.superDecoder(forKey: key)).container(keyedBy: type) - } - func nestedUnkeyedContainer(forKey key: Key) throws -> UnkeyedDecodingContainer { - try OwnedDecoder(base: base.superDecoder(forKey: key)).unkeyedContainer() - } - func superDecoder() throws -> Decoder { OwnedDecoder(base: try base.superDecoder()) } - func superDecoder(forKey key: Key) throws -> Decoder { OwnedDecoder(base: try base.superDecoder(forKey: key)) } -} - -private struct OwnedUnkeyed: UnkeyedDecodingContainer { - var base: UnkeyedDecodingContainer - var codingPath: [CodingKey] { base.codingPath } - var count: Int? { base.count } - var isAtEnd: Bool { base.isAtEnd } - var currentIndex: Int { base.currentIndex } - mutating func decodeNil() throws -> Bool { try base.decodeNil() } - mutating func decode(_ type: Bool.Type) throws -> Bool { try base.decode(type) } - mutating func decode(_ type: String.Type) throws -> String { try base.decode(type) } - mutating func decode(_ type: Double.Type) throws -> Double { try base.decode(type) } - mutating func decode(_ type: Float.Type) throws -> Float { try base.decode(type) } - mutating func decode(_ type: Int.Type) throws -> Int { try base.decode(type) } - mutating func decode(_ type: Int8.Type) throws -> Int8 { try base.decode(type) } - mutating func decode(_ type: Int16.Type) throws -> Int16 { try base.decode(type) } - mutating func decode(_ type: Int32.Type) throws -> Int32 { try base.decode(type) } - mutating func decode(_ type: Int64.Type) throws -> Int64 { try base.decode(type) } - mutating func decode(_ type: UInt.Type) throws -> UInt { try base.decode(type) } - mutating func decode(_ type: UInt8.Type) throws -> UInt8 { try base.decode(type) } - mutating func decode(_ type: UInt16.Type) throws -> UInt16 { try base.decode(type) } - mutating func decode(_ type: UInt32.Type) throws -> UInt32 { try base.decode(type) } - mutating func decode(_ type: UInt64.Type) throws -> UInt64 { try base.decode(type) } - mutating func decode(_ type: Value.Type) throws -> Value { - if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { - return try base.decode(type) - } - return try base.decode(OwnedValue.self).value - } - mutating func nestedContainer(keyedBy type: Nested.Type) throws -> KeyedDecodingContainer { - try OwnedDecoder(base: base.superDecoder()).container(keyedBy: type) - } - mutating func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { - try OwnedDecoder(base: base.superDecoder()).unkeyedContainer() - } - mutating func superDecoder() throws -> Decoder { OwnedDecoder(base: try base.superDecoder()) } -} - -private struct OwnedSingle: SingleValueDecodingContainer { - let base: SingleValueDecodingContainer - let decoder: OwnedDecoder - var codingPath: [CodingKey] { base.codingPath } - func decodeNil() -> Bool { base.decodeNil() } - func decode(_ type: Bool.Type) throws -> Bool { try base.decode(type) } - func decode(_ type: String.Type) throws -> String { try base.decode(type) } - func decode(_ type: Double.Type) throws -> Double { try base.decode(type) } - func decode(_ type: Float.Type) throws -> Float { try base.decode(type) } - func decode(_ type: Int.Type) throws -> Int { try base.decode(type) } - func decode(_ type: Int8.Type) throws -> Int8 { try base.decode(type) } - func decode(_ type: Int16.Type) throws -> Int16 { try base.decode(type) } - func decode(_ type: Int32.Type) throws -> Int32 { try base.decode(type) } - func decode(_ type: Int64.Type) throws -> Int64 { try base.decode(type) } - func decode(_ type: UInt.Type) throws -> UInt { try base.decode(type) } - func decode(_ type: UInt8.Type) throws -> UInt8 { try base.decode(type) } - func decode(_ type: UInt16.Type) throws -> UInt16 { try base.decode(type) } - func decode(_ type: UInt32.Type) throws -> UInt32 { try base.decode(type) } - func decode(_ type: UInt64.Type) throws -> UInt64 { try base.decode(type) } - func decode(_ type: Value.Type) throws -> Value { - if type == Data.self || type == Date.self || type == URL.self || type == Decimal.self { - return try base.decode(type) - } - return try Value(from: decoder) - } -} diff --git a/swift/JournalList.swift b/swift/JournalList.swift index 07ecad15..c438bfeb 100644 --- a/swift/JournalList.swift +++ b/swift/JournalList.swift @@ -86,7 +86,7 @@ import Observation let positions: [String: Int] init?(_ json: String) { - guard let properties = try? JournalInternalJSON.decode(Properties.self, from: Data(json.utf8)) + guard let properties = try? JSONDecoder().decode(Properties.self, from: Data(json.utf8)) else { return nil } var positions: [String: Int] = [:] var keys: Set = [] diff --git a/swift/JournalLocalAccountBindingStore.swift b/swift/JournalLocalAccountBindingStore.swift index d0a1e9b4..efbb606f 100644 --- a/swift/JournalLocalAccountBindingStore.swift +++ b/swift/JournalLocalAccountBindingStore.swift @@ -6,7 +6,6 @@ enum JournalLocalAccountBindingStore { private static let maximumBytes = 4_096 private struct Binding: Codable { let version: Int; let userId: String; let managedSyncOrigin: String } - // Version projection only; Binding below admits the complete closed record. private struct Format: Decodable { let version: Int } static func query() -> [CFString: Any] { @@ -46,7 +45,7 @@ enum JournalLocalAccountBindingStore { } let binding: Binding do { - binding = try JournalInternalJSON.decode(Binding.self, from: data) + binding = try JSONDecoder().decode(Binding.self, from: data) _ = try encode(userID: binding.userId, managedSyncOrigin: binding.managedSyncOrigin) } catch { throw JournalAccountIssue(kind: .corruptBinding, stage: .bindingRead, formatVersion: version) diff --git a/swift/JournalNativeServices.swift b/swift/JournalNativeServices.swift index 39cb652d..543721a5 100644 --- a/swift/JournalNativeServices.swift +++ b/swift/JournalNativeServices.swift @@ -102,7 +102,7 @@ private final class JournalNativeAccountStorage: Sendable { } let recovery: Recovery do { - recovery = try JournalInternalJSON.decode(Recovery.self, from: bytes) + recovery = try JSONDecoder().decode(Recovery.self, from: bytes) guard recovery.version == 1 else { throw JournalPlatformServices.Failure.unavailable } } catch { throw JournalAccountIssue(kind: .unavailable, stage: .bindingWrite) diff --git a/swift/JournalPlatformWire.swift b/swift/JournalPlatformWire.swift index af5f0764..12a29236 100644 --- a/swift/JournalPlatformWire.swift +++ b/swift/JournalPlatformWire.swift @@ -153,7 +153,7 @@ enum JournalPlatformWire { } private static func decode(_ type: Value.Type, _ payload: Data) throws -> Value { - do { return try JournalInternalJSON.decode(type, from: payload) } + do { return try JSONDecoder().decode(type, from: payload) } catch { throw Failure.invalidPacket } } diff --git a/test/dune b/test/dune index 646d29e4..ad8e0bc4 100644 --- a/test/dune +++ b/test/dune @@ -124,7 +124,6 @@ (deps test_wrapped_graph_key_crypto.sh wrapped_graph_key_crypto_test.swift - ../swift/JournalInternalJSON.swift ../swift/JournalE2EECrypto.swift) (action (run sh %{dep:test_wrapped_graph_key_crypto.sh} %{workspace_root}))) diff --git a/test/test_wrapped_graph_key_crypto.sh b/test/test_wrapped_graph_key_crypto.sh index 6f7317bc..e557eb65 100644 --- a/test/test_wrapped_graph_key_crypto.sh +++ b/test/test_wrapped_graph_key_crypto.sh @@ -8,7 +8,6 @@ trap 'rm -f "$binary"' EXIT HUP INT TERM swiftc \ -swift-version 6 \ -D DEBUG \ - "$root/swift/JournalInternalJSON.swift" \ "$root/swift/JournalE2EECrypto.swift" \ "$root/test/wrapped_graph_key_crypto_test.swift" \ -o "$binary" diff --git a/tool/build_authentication_probe.py b/tool/build_authentication_probe.py index 91f4e5c5..e2c83fe3 100644 --- a/tool/build_authentication_probe.py +++ b/tool/build_authentication_probe.py @@ -25,7 +25,7 @@ 'UIInterfaceOrientationLandscapeLeft', 'UIInterfaceOrientationLandscapeRight'], })) sources = [root/'swift/JournalAuthentication.swift', root/'swift/JournalAuthenticationView.swift', - root/'swift/JournalInternalJSON.swift', root/'swift/JournalPlatformWire.swift', root/'swift/JournalPlatformServices.swift', + root/'swift/JournalPlatformWire.swift', root/'swift/JournalPlatformServices.swift', root/'swift/JournalLocalAccountBindingStore.swift', root/'apple-tests/authentication/JournalAuthenticationProbe.swift'] sdk = subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', '--show-sdk-path'], text=True).strip() diff --git a/tool/test_asset_interop.py b/tool/test_asset_interop.py index 29eca3c0..bb23c5b3 100644 --- a/tool/test_asset_interop.py +++ b/tool/test_asset_interop.py @@ -22,7 +22,7 @@ print(work, flush=True) record = {"commands": [], "sources": {}, "result": "incomplete"} for source in [crypt, upstream / "deps/db/src/logseq/db/sqlite/util.cljs", - root / "swift/JournalInternalJSON.swift", root / "swift/JournalE2EECrypto.swift", + root / "swift/JournalE2EECrypto.swift", root / "logseq_sync/lib/effect_runner/protocol/asset_codec.ml", root / "logseq_sync/test/asset_native_interop.ml", root / "logseq_sync/test/asset_adapter_memory.ml", @@ -67,7 +67,7 @@ def run(command, env=None): run(["swiftc", "--version"]) run(nbb + ["generate", str(work)]) library = work / "libJournalAssetCrypto.dylib" -run(["swiftc", "-emit-library", str(root / "swift/JournalInternalJSON.swift"), str(root / "swift/JournalE2EECrypto.swift"), +run(["swiftc", "-emit-library", str(root / "swift/JournalE2EECrypto.swift"), "-o", str(library)]) run(["dune", "build", "logseq_sync/test/asset_native_interop.exe", "logseq_sync/test/asset_adapter_memory.exe"]) diff --git a/tool/test_json_boundaries.py b/tool/test_json_boundaries.py deleted file mode 100644 index cc7e8fc3..00000000 --- a/tool/test_json_boundaries.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public native owner and internal decoder tests; synthetic bytes, no App or credentials.""" -from pathlib import Path -import subprocess -import tempfile - -root = Path(__file__).resolve().parents[1] -with tempfile.TemporaryDirectory(prefix='journal-json-boundaries-') as directory: - output = Path(directory) / 'tests' - sources = ['JournalInternalJSON.swift', 'JournalPlatformWire.swift', - 'JournalPlatformServices.swift', 'JournalLocalAccountBindingStore.swift', - 'JournalCognitoOAuth.swift'] - subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', - str(Path(directory)/'modules'), - *[str(root/'swift'/name) for name in sources], - str(root/'apple-tests/platform-wire/JournalJSONBoundaryTests.swift'), - '-o',str(output)],check=True) - subprocess.run([str(output)],check=True,timeout=60) diff --git a/tool/test_swiftui_cognito.py b/tool/test_swiftui_cognito.py index 9182bafd..9ff9abd7 100644 --- a/tool/test_swiftui_cognito.py +++ b/tool/test_swiftui_cognito.py @@ -5,7 +5,7 @@ root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix='journal-cognito-') as directory: destination = Path(directory) - sources = ['JournalInternalJSON.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', + sources = ['JournalPlatformWire.swift', 'JournalPlatformServices.swift', 'JournalAuthentication.swift', 'JournalCognitoOAuth.swift', 'JournalCognitoSession.swift'] subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-Xfrontend', '-disable-sandbox', '-module-cache-path', str(destination/'modules'), *[str(root/'swift'/s) for s in sources], diff --git a/tool/test_swiftui_events.py b/tool/test_swiftui_events.py index c8fcff2e..4a09eccf 100644 --- a/tool/test_swiftui_events.py +++ b/tool/test_swiftui_events.py @@ -6,8 +6,8 @@ root = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory(prefix='journal-platform-events-') as directory: executable = Path(directory) / 'events-tests' - subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), - str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), + subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', + str(root / 'swift/JournalPlatformWire.swift'), str(root / 'swift/JournalPlatformEvents.swift'), str(root / 'apple-tests/platform-events/JournalPlatformEventsTests.swift'), '-o', str(executable)], check=True) diff --git a/tool/test_swiftui_platform.py b/tool/test_swiftui_platform.py index f866508b..052aaebb 100644 --- a/tool/test_swiftui_platform.py +++ b/tool/test_swiftui_platform.py @@ -58,7 +58,7 @@ def run(directory, command): *[source.stem + '.cmx' for source in sources if source.suffix == '.ml']]) run(destination, [str(destination / 'wire_ocaml'), 'emit', 'requests.json']) run(destination, ['swiftc', '-swift-version', '6', '-module-cache-path', str(destination / 'module-cache'), '-o', 'wire_swift', - str(ROOT / 'swift/JournalInternalJSON.swift'), str(ROOT / 'swift/JournalPlatformWire.swift'), + str(ROOT / 'swift/JournalPlatformWire.swift'), str(ROOT / 'swift/JournalPlatformServices.swift'), str(ROOT / 'swift/JournalStartupConfiguration.swift'), str(ROOT / 'swift/JournalLocalAccountBindingStore.swift'), diff --git a/tool/test_swiftui_services.py b/tool/test_swiftui_services.py index b097aaaa..88822e0f 100644 --- a/tool/test_swiftui_services.py +++ b/tool/test_swiftui_services.py @@ -7,7 +7,7 @@ with tempfile.TemporaryDirectory(prefix='journal-platform-services-') as directory: executable = Path(directory)/'services-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), - str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), + str(root/'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'apple-tests/platform-services/JournalPlatformServicesTests.swift'), '-o', str(executable)], check=True) @@ -15,7 +15,7 @@ recovery = Path(directory)/'recovery-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-Xfrontend', '-disable-sandbox', '-module-cache-path', str(Path(directory)/'modules'), - str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), + str(root/'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalLocalAccountBindingStore.swift'), str(root/'apple-tests/platform-services/JournalAccountRecoveryTests.swift'), @@ -23,7 +23,7 @@ subprocess.run([str(recovery)], check=True, timeout=60) threading = Path(directory)/'account-threading-tests' subprocess.run(['xcrun', 'swiftc', '-swift-version', '6', '-module-cache-path', str(Path(directory)/'modules'), - str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), + str(root/'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalStartupConfiguration.swift'), str(root/'swift/JournalNativeServices.swift'), @@ -32,7 +32,7 @@ subprocess.run([str(threading)], check=True, timeout=60) cold = Path(directory)/'cold-tests' subprocess.run(['xcrun','swiftc','-swift-version','6','-module-cache-path',str(Path(directory)/'modules'), - str(root / 'swift/JournalInternalJSON.swift'), str(root / 'swift/JournalPlatformWire.swift'), + str(root/'swift/JournalPlatformWire.swift'), str(root/'swift/JournalPlatformServices.swift'), str(root/'swift/JournalStartupConfiguration.swift'), str(root/'swift/JournalNativeServices.swift'), diff --git a/tool/test_swiftui_warm_start.py b/tool/test_swiftui_warm_start.py index 06b32d75..9266ab0c 100644 --- a/tool/test_swiftui_warm_start.py +++ b/tool/test_swiftui_warm_start.py @@ -97,7 +97,7 @@ def run(command, cwd=host): generator.write_text(bootstrap+'\n#load '+json.dumps(str(fixture_source.with_suffix('.cmo')))+';;\n') sources = [root/'swift'/name for name in - ['JournalInternalJSON.swift', 'JournalE2EECrypto.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', + ['JournalE2EECrypto.swift', 'JournalPlatformWire.swift', 'JournalPlatformServices.swift', 'JournalStartupConfiguration.swift', 'JournalChrome.swift']] source = root/'apple-tests/warm-start/JournalWarmStartAcceptance.swift' for tracked in sources + [source]: From a5b58d5b511cfe0fb588619510531b5d378c7ea9 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 14 Oct 2026 18:14:19 +0800 Subject: [PATCH 4/4] Scope retained root feedback test lookups to navigation owner --- .../2026-10-14-external-input-open-json.md | 4 ++++ test/application_view_test.ml | 20 ++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md index 13c04a33..efd18ab0 100644 --- a/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md +++ b/docs/agent-guide/implemented/bugfix/2026-10-14-external-input-open-json.md @@ -10,6 +10,8 @@ Limit this PR to OCaml JSON codecs, as explicitly requested by the user. Keep PP All Swift codec behavior and its compiler/test/CI configuration are restored exactly to the PR's merge-base. The generic Swift decoder, native call-site replacements, native boundary tests and offline provider compile mode are outside the narrowed scope. +The user additionally approved the smallest test-only correction for the existing CI failure in Application's retained root feedback checks. The fixture's global title lookup can select Detail's same-title node instead of the root node. Add an optional identity predicate to that test helper and restrict root-content observations to the root navigation owner; retain the original layout and node identity assertions. This does not change production UI behavior. + ## Alternatives considered ### Declare only tx-id @@ -43,3 +45,5 @@ Tests were written first. Against unchanged main953, public WS extension/declare Frozen upstream dev 949a4f0e72cddb2dad34091393bfa09592171944 declares optional non-null UUID tx-id in server entries; old 015cc9f does not declare that server field. Both use open Malli maps. This source evidence does not establish the screenshot value/type or deployment revision: [frozen schema](https://github.com/logseq/logseq/blob/949a4f0e72cddb2dad34091393bfa09592171944/deps/db-sync/src/logseq/db_sync/malli_schema.cljs#L10). Journal does not consume tx-id. After scope removal, rerun the OCaml build/native embed and regression suite, review the exact new head in the existing independent Codex session using journal-code-review, and track its CI to success. Prior Swift-inclusive review/CI snapshots are superseded. spec-dev-tool check --all has three preexisting invalid historical decisions unrelated to this change. + +The unchanged root-feedback assertion was reproduced on both 953d4a7 and 9293bad by randomizing only the test observer table's hash order. The root body has one stack candidate; both snapshots preserve root content node 180, while the global title lookup may select Detail node 259. The candidate correction passes the two fixed public Application cases in five randomized runs per snapshot (20 case executions). All original containment and node identity comparisons remain. Diagnostic mirrors and logs stay outside the repository. diff --git a/test/application_view_test.ml b/test/application_view_test.ml index 7d6ade2b..f2eaacc0 100644 --- a/test/application_view_test.ml +++ b/test/application_view_test.ml @@ -1176,10 +1176,10 @@ let run_favorites_native_visibility ((key, member "value" op) :: List.remove_assoc key previous) | _ -> ()) in - let find key value = + let find ?(accept = fun _ -> true) key value = let direct = Hashtbl.fold (fun id values found -> - if List.assoc_opt key values = Some (`String value) then Some id else found) + if accept id && List.assoc_opt key values = Some (`String value) then Some id else found) props None in @@ -1189,7 +1189,7 @@ let run_favorites_native_visibility match List.assoc_opt "payload" values with | Some (`String payload) -> let p = Yojson.Safe.from_string payload in - if (List.mem (Yojson.Safe.Util.member "mode" p) [`String "detail-text"; `String "bounded-text"]) + if accept id && (List.mem (Yojson.Safe.Util.member "mode" p) [`String "detail-text"; `String "bounded-text"]) && Yojson.Safe.Util.member "text" p = `String value then Some id else found | _ -> found) props None | None -> None @@ -1786,19 +1786,23 @@ let run_favorites_native_visibility ~some:inside_body (Hashtbl.find_opt parents id) in + let root_content = + find ~accept:(fun id -> navigation_page id = Some (`String "root")) + "text" "Timeline fixture 1" |> Option.get + in Alcotest.(check bool) "root feedback reserves space for the retained page content" true - (inside_body (Option.get (find "text" "Timeline fixture 1"))); + (inside_body root_content); Alcotest.(check int) "showing feedback preserves the original root content node" original_root_content - (Option.get (find "text" "Timeline fixture 1")); + root_content; Alcotest.(check bool) "failed import keeps its own staged file" true (Sys.file_exists (List.hd !staged_paths)); - let retained_root = Option.get (find "text" "Timeline fixture 1") in + let retained_root = root_content in back (); Alcotest.(check int) "Back retains the root feedback owner and its unresolved error" @@ -1807,7 +1811,9 @@ let run_favorites_native_visibility Alcotest.(check int) "feedback updates retain the root content" retained_root - (Option.get (find "text" "Timeline fixture 1")); + (find + ~accept:(fun id -> navigation_page id = Some (`String "root")) + "text" "Timeline fixture 1" |> Option.get); enter 0; dispatch (Lui_protocol.Press detail_actions); settle_worker ();