From ee00a2aa07f85944aa1adf74553854d9511fc60e Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 10:56:31 +0800 Subject: [PATCH 1/7] Add reusable Apple build with explicit native provenance --- AGENTS.md | 51 ++ .../2026-10-15-reusable-apple-build.md | 42 + tool/build_journal_apple.sh | 220 +---- tool/journal_build.py | 796 ++++++++++++++++++ tool/package_journal_apple_icons.sh | 5 +- tool/test_apple_app_icon.py | 2 +- tool/test_journal_build.py | 312 +++++++ 7 files changed, 1207 insertions(+), 221 deletions(-) create mode 100644 docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md create mode 100644 tool/journal_build.py create mode 100644 tool/test_journal_build.py diff --git a/AGENTS.md b/AGENTS.md index 7d1e4193..8b14dbd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,3 +9,54 @@ - If development is blocked because the `.mli` definitions under `spec/` are unclear or unreasonable, stop development immediately and report the specific spec issue, suggested changes, and rationale. - Do not modify any ocaml files in bonsai_flutter repo - Prefer native bonsai-ui/SwiftUI components and their built-in layout, scrolling, and animation behavior. Do not recreate these behaviors using per-row geometry tracking, shared observable geometry state, or repeated collection scans during scrolling. If native APIs cannot satisfy a requirement, explain the limitation and obtain explicit approval before implementing custom UI coordination. + +## Build + +Use `bash tool/build_journal_apple.sh --help` as the Apple build entry point. It uses the existing Dune native object, Swift package and icon packager; it never installs an app, touches a graph, changes pins, downloads dependencies, or clears caches. Provision dependencies separately in an explicitly chosen switch using the existing CI lock/resolved-pin workflow (`tool/prepare_ci_dependencies.py`); that workflow fetches and pins dependencies and must not be run against another task's shared switch without authorization. + +| Target | SDK / architecture | Current support | +| --- | --- | --- | +| `macos` | `macosx` / arm64 | Native OCaml build through an explicit matching switch; default signing is adhoc. | +| `ios-simulator --link-only` | `iphonesimulator` / arm64 | Explicit stub: Swift/link/package/signature checks only; no runnable OCaml acceptance. | +| iOS `--experimental-host-restamp --native-manifest FILE` | selected `iphonesimulator` or `iphoneos` / arm64 | Explicit compatibility mode: verified producer host OCaml object, freshly compiled target DNS/crypto C and Swift. Full OCaml/runtime iOS ABI remains unvalidated. | +| iOS without either explicit mode | distinct Simulator / iPhoneOS ABI | Refused; no implicit stub or host restamp. | + +Run from this worktree; choose an external cache and a local LUI `platform/apple` checkout. For native builds its clean Git HEAD must equal the LUI pin in `logseq_journal.opam.locked`. + +```sh +bash tool/build_journal_apple.sh macos --switch YOUR_SWITCH \ + --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache --preflight +bash tool/build_journal_apple.sh macos --switch YOUR_SWITCH \ + --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache +bash tool/build_journal_apple.sh ios-simulator --link-only \ + --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache +bash tool/build_journal_apple.sh ios-device --experimental-host-restamp \ + --native-manifest ../native-input.json --lui-package /path/to/lui/platform/apple \ + --cache-dir ../journal-build-cache --sign-identity CERTIFICATE_SHA1 \ + --provisioning-profile /path/to/device.mobileprovision \ + --sign-entitlements /path/to/device.entitlements --build-number 1.2.3 +python3 tool/test_journal_build.py +``` + +The experimental input is a small envelope referencing existing producer records by SHA-256. Paths below are relative to the envelope; producer records retain their explicit input paths. Keep these records and the fixed complete object read-only. + +```json +{ + "schema_version": 1, + "source": {"path": "preflight.json", "sha256": "SHA256"}, + "link": {"path": "native-command-manifest.json", "sha256": "SHA256"}, + "xmlm_force_link": {"path": "xmlm-force-link-proof.json", "sha256": "SHA256"}, + "validated_build": {"path": "successful-build/manifest.json", "sha256": "SHA256"} +} +``` + +`source` binds commit/LUI/Signal/DataScript, production source hashes and the host complete-object hash. `link` supplies compiler/version, archive/stub input hashes and pinned mldoc/XMLm provenance. `xmlm_force_link` describes only `ocamlopt -a -linkall -o OWN/xmlm.cmxa OWN/xmlm.cmx`, with unchanged original cmx/object hashes. The producer commit must be an ancestor of the current builder revision, and the complete native source set plus Dune/opam/link configuration must still match its bytes exactly. Thus Build/tool/docs commits can advance HEAD; native changes cannot silently reuse an older object. Reports keep both current `source_sha` and `native_source_sha`. The importer never executes recorded commands. + +`validated_build` is optional and must reference a trusted successful experimental build manifest from this entry point, with the same original three producer records, native source, compiler identity and complete-object hash. Without it, a first import verifies all still-present producer files and XMLm bytes/metadata. With it, already-consumed archives/compiler/source paths are historical provenance, so subsequent builds may change or remove them without invalidating the fixed object. Their original hashes remain recorded; this does not prove that the original object can still be rebuilt from those paths. Current object bytes/platform/symbols, exact native source/configuration/pins, runtime headers used by fresh target C, evidence/receipt bytes, SDK and Swift/signing inputs remain checked. Never edit an original producer record or invent a successful receipt to bypass a mismatch. Changed native bytes require matching new producer evidence. + +- `--preflight` validates without compiling. The Dune route compares the explicit switch with locks/pins; the imported route validates producer source/dependency/compiler hashes instead of requiring its private dependencies to be globally installed. Neither repairs pins or installs dependencies. Moving refs have no network freshness claim. +- Every run creates an external `run-*/manifest.json` and phase logs. Only `status: built` confirms a completed artifact; `preflight-passed` and `link-only` do not establish runtime behavior. The manifest records exact source SHA plus dirty input hashes, dependency/toolchain/SDK/target/flags, native input and final bundle hashes, signing and version. Keep it with the app when handing off for installation. +- Source, actual dependency native ABI inputs, compiler/Xcode/SDK, flags or target changes select another cache namespace. Historical producer identities remain bound to the imported object instead of the present contents of consumed archive paths. Identical inputs reuse Dune/Swift incremental state. Nothing is automatically deleted. The cache root has a nonblocking process lock; native builds also hold the opam switch read lock. Do not point at another task's legacy build/cache/signing directories; coordinate with its owner first. The script owns only its external cache and new output directory; it refuses an existing `--app-dir` instead of removing it. +- `--configuration debug|release`, `--deployment-target` (>=26.0), `--jobs`, `--version` and `--build-number` are explicit inputs. Unset ambient `SDKROOT`/compiler/OCaml lookup overrides; the Swift manifest runs on macOS and the selected target SDK is passed explicitly. Production native objects must retain mldoc and XMLm symbols: merely listing an unused `.cmxa` does not prove retention. A missing unit stops the build for linker-rule review; ask before changing protected Dune rules. +- Resources and C4 icons are packaged before signing. Device mode checks the existing iOS profile, expiration, certificate, team/bundle and allowed entitlements, then reads back the actual signature/entitlements. Simulator uses adhoc signing with entitlements in `__TEXT`; macOS development signing takes `--sign-identity` and `JOURNAL_MACOS_TEAM_ID`. Version flags affect only the generated plist. Build never installs or claims startup acceptance. +- Experimental mode checks every Swift object against the selected Mach-O platform, checks the final DNS/crypto link-map provider is its fresh target C object, and requires runtime/mldoc/XMLm symbols. Its report and generated plist explicitly mark host-restamped runtime origin and `cross_abi_validated=false`. This reproduces the existing limited route, not full OCaml iOS cross-compilation. Raw `JOURNAL_OCAML_OBJECT`/extra-object injection and the old object-injecting Dune aliases remain refused; use the explicit commands above. Any protected Dune/spec/lock edit still needs separate approval. diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md new file mode 100644 index 00000000..42aabcc7 --- /dev/null +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -0,0 +1,42 @@ +# Reusable Apple Build + +## Problem + +Recent Journal builds repeatedly reconstructed dependency, target, cache and signing checks in external task scripts. The existing Apple entry point defaults to a stub, suppresses signing failures and does not record artifact provenance. + +## Decision + +Extend `tool/build_journal_apple.sh` through a Python companion using the existing Swift package and icon packager. Build the production macOS native object through Dune in an owned external cache after read-only switch/lock/pin checks. Strict default iOS builds still require explicit mode selection. The user has additionally authorized `--experimental-host-restamp --native-manifest FILE`: import only a hash-verified producer host object matching current native sources, validate its single-unit XMLm force-link proof, compile DNS/crypto C freshly for the selected target, and reuse the common Swift/package/sign pipeline. The input manifest references the existing source, link and XMLm proof JSON files by path/hash; it contains no executable command hooks. Device signing inputs are explicit. Record the host runtime ABI limitation; build never installs or changes pins. + +After one successful import, an optional hash-bound `validated_build` receipt proves the original producer records and consumed input hashes for the immutable complete object. Historical archives are no longer live build inputs: their later modification/removal cannot alone invalidate that object. Current object/native source/configuration/pins, runtime C headers, original evidence/receipt, SDK identity, Swift and signing inputs remain strict. The producer commit must be an ancestor of the builder revision; exact native bytes/configuration still match across Build/tool/docs commits, and reports distinguish these revisions. The original producer records are never rewritten. The build importer owns this regression; no application reducer can reproduce a filesystem/provenance contract failure, so tests exercise the importer with a tiny real Mach-O producer and temporary external Git/source files. + +## Alternatives considered + +### Alternative + +Do not copy historical device deployment scripts: their host ARM64 restamp route is not a verified iOS OCaml cross toolchain. Do not add a separate competing build entry point or change protected Dune/spec/lock files. + +## Acceptance criteria + +- CLI failure checks prevent accidental stub production builds, unsupported targets, unsafe output replacement and concurrent cache use. +- Read-only preflight identifies actual installed dependency mismatches without changing the switch. +- Explicit link-only assembly verifies target, resources, version and signature, with manifest and phase logs outside Git. +- Input changes select a new cache without clearing previous caches; repeated identical builds reuse their namespace. +- Explicit experimental builds validate producer source/configuration/dependency identities, retain runtime/mldoc/XMLm, compile target C, check Swift object platforms, and verify the packaged device signature without claiming full OCaml iOS ABI acceptance. + +## Risks + +- Current local switch lacks mldoc and XMLm pins; native Dune builds may be blocked rather than repaired implicitly. The explicit imported-native route reuses producer dependency provenance instead of constructing another switch. +- Host runtime restamping remains experimental, not full target-native OCaml compilation or ABI validation. Production iOS C and Swift targets must be independently checked; raw object injection remains blocked. Link-only is not runnable acceptance. + +## Consequences + +- The existing shell entry point delegates to one standard-library Python implementation. The documented entry point replaces legacy object-injecting aliases for supported builds; protected Dune/spec/lock files are unchanged. +- Eighteen boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. +- An actual experimental iPhoneOS build produced a signed production App: runtime/mldoc/XMLm retained, 55 Swift objects verified as iOS ARM64, DNS/crypto link-map providers verified as freshly compiled iPhoneOS C, and profile/signature/entitlements read back successfully. Subsequent review tightened source deletion/configuration, compiler and profile guards. A final retry correctly stopped before compilation when another authorized task changed 14 historical producer archive inputs; it did not accept the old provenance or interfere with that cache. The successful App remains preserved. Independent journal-code-review found no remaining substantive issue after fixes. +- The later archive mismatch exposed an overly strict importer lifecycle rule: the fixed complete object remained valid while another owner normally updated its consumed archives. Receipt-bound historical provenance now distinguishes that state from changed current inputs. After the repair, a final production iPhoneOS build passed through this entry point: fixed OCaml complete object reused, target C and all Swift/LUI objects compiled in a new cache namespace, icons/resources packaged, signature/entitlements and input stability verified. No OCaml recompilation or phone operation occurred. +- Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. + +## Questions + +- The delegated user instruction supplies the target/safety scope and explicitly requests implementation, verification and independent review. No unresolved product choice requires another approval. Protected Dune/spec/lock changes remain outside scope. diff --git a/tool/build_journal_apple.sh b/tool/build_journal_apple.sh index 50801518..d4756d19 100755 --- a/tool/build_journal_apple.sh +++ b/tool/build_journal_apple.sh @@ -1,221 +1,5 @@ #!/usr/bin/env bash -# Builds the journal Apple host (swift/Package.swift -> JournalApp) with the -# OCaml runtime linked in. -# -# Usage: tool/build_journal_apple.sh [--app-dir DIR] -# -# Native link inputs: -# * journal_lui_bridge.o is always compiled here from app/journal_lui_bridge.c -# against the OCaml headers of the target toolchain — as a compile check. -# It is NOT a link input: the dune-produced complete object already folds -# in the app's foreign_stubs copy, and the link stub below redefines all -# entries. Set JOURNAL_EXTRA_OBJECTS (colon-separated) to append extra -# objects if a producer ever emits an object without the bridge stubs. -# * The OCaml complete object (the `native_embed` product) is NOT built by -# this script — it is produced by the dune/opam side of the workspace. -# Point JOURNAL_OCAML_OBJECT at the artifact: -# macOS: dune builds app/native_embed.exe.o with the host switch. -# iOS simulator: the shared cross toolchain at -# ${LG_IOS_OCAML_PREFIX:-$OPAMROOT/lg-ocaml-toolchains/ocaml-/targets/arm64-apple-ios-simulator} -# must have produced a complete object for the target triple; pass its -# path through JOURNAL_OCAML_OBJECT. -# Without JOURNAL_OCAML_OBJECT the script links a stub object so the Swift -# side still verifies end-to-end (link only — the binary is not runnable). -# -# Reuses the lui mobile conventions: LG_IOS_OCAML_PREFIX / -# $OPAMROOT/lg-ocaml-toolchains (as in lui/tooling/mobile/build_components_*.sh). - +# One Apple build entry point; never installs, updates dependencies or clears caches. set -euo pipefail - repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -# Probe harnesses (tool/lui_probe_host.py) stage an overlaid copy of swift/ + a -# probe Info.plist under a temp host and point these overrides at it. -swift_dir=${JOURNAL_SWIFT_DIR:-$repo_root/swift} -info_plist=${JOURNAL_INFO_PLIST:-$repo_root/apple/Info.plist} -entitlements_dir=${JOURNAL_ENTITLEMENTS_DIR:-$repo_root/config/entitlements} -platform=${1:-macos} -app_dir_arg=${2:-} -ocaml_version=${LG_OCAML_VERSION:-5.5.0} -opam_root=${OPAMROOT:-$(opam var root --safe 2>/dev/null || echo "$HOME/.opam")} -shared_root=${LG_OCAML_TOOLCHAIN_ROOT:-$opam_root/lg-ocaml-toolchains} - -case "$platform" in - macos) - deployment_target=${JOURNAL_MACOS_DEPLOYMENT_TARGET:-26.0} - triple="arm64-apple-macosx${deployment_target}" - sdk_path=$(xcrun --sdk macosx --show-sdk-path) - clang=$(xcrun --sdk macosx --find clang) - # Host switch provides headers + runtime for the bridge C file. - ocaml_prefix=${JOURNAL_OCAML_PREFIX:-$(ocamlfind printconf destdir 2>/dev/null | sed 's|/lib$||' || true)} - [[ -n $ocaml_prefix ]] || ocaml_prefix="$opam_root/default" - ocaml_include="$ocaml_prefix/lib/ocaml" - ;; - ios-simulator) - deployment_target=${JOURNAL_IOS_DEPLOYMENT_TARGET:-26.0} - triple="arm64-apple-ios${deployment_target}-simulator" - sdk_path=$(xcrun --sdk iphonesimulator --show-sdk-path) - clang=$(xcrun --sdk iphonesimulator --find clang) - target_prefix=${LG_IOS_OCAML_PREFIX:-$shared_root/ocaml-$ocaml_version/targets/$triple} - if [[ -d $target_prefix/lib/ocaml ]]; then - ocaml_include="$target_prefix/lib/ocaml" - else - # journal_lui_bridge.c is a compile check only (not a link input); the - # host OCaml headers are platform-independent for it. - ocaml_prefix=${JOURNAL_OCAML_PREFIX:-$(ocamlfind printconf destdir 2>/dev/null | sed 's|/lib$||' || true)} - [[ -n $ocaml_prefix ]] || ocaml_prefix="$opam_root/default" - ocaml_include="$ocaml_prefix/lib/ocaml" - fi - ;; - *) echo "usage: $0 " >&2; exit 2 ;; -esac - -build_dir="$repo_root/_build/apple/$platform" -mkdir -p "$build_dir" - -# --- journal_lui_bridge.o ------------------------------------------------- -"$clang" \ - -target "$triple" \ - -isysroot "$sdk_path" \ - -fPIC \ - -I "$ocaml_include" \ - -c "$repo_root/app/journal_lui_bridge.c" \ - -o "$build_dir/journal_lui_bridge.o" - -# --- OCaml complete object ------------------------------------------------ -ocaml_object=${JOURNAL_OCAML_OBJECT:-} -if [[ -z $ocaml_object ]]; then - echo "warning: JOURNAL_OCAML_OBJECT unset; linking a stub object (link-only build)" >&2 - cat > "$build_dir/ocaml_stub.c" <<'STUB' -// Link-validation stub matching app/journal_lui_bridge.c's exported entries. -// Replace via JOURNAL_OCAML_OBJECT for a runnable binary. -#include -typedef void (*patch_cb)(const char *); -typedef void (*wakeup_cb)(void); -typedef void (*platform_request_cb)(const char *, int32_t); -int32_t lui_ocaml_start(patch_cb cb, int32_t p, int32_t h, const char *d, int32_t l) - { (void)p; (void)h; (void)d; (void)l; if (cb) cb(""); return 1; } -int32_t lui_ocaml_stop(void) { return 1; } -int32_t lui_ocaml_appear(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_press(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_long_press(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_text_changed(int64_t n, const char *t) { (void)n; (void)t; return 1; } -int32_t lui_ocaml_submit(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_dismiss(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_double_press(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_toggle_changed(int64_t n, int32_t c) { (void)n; (void)c; return 1; } -int32_t lui_ocaml_radio_changed(int64_t n) { (void)n; return 1; } -int32_t lui_ocaml_slider_changed(int64_t n, double v) { (void)n; (void)v; return 1; } -int32_t journal_ocaml_extension_event(int64_t n, const char *name, const char *p) - { (void)n; (void)name; (void)p; return 1; } -int32_t journal_ocaml_pump(void) { return 1; } -void journal_ocaml_platform_event(const char *d, int32_t l) { (void)d; (void)l; } -void journal_ocaml_platform_response(const char *d, int32_t l) { (void)d; (void)l; } -void journal_ocaml_set_wakeup_callback(wakeup_cb cb) { (void)cb; } -void journal_ocaml_set_platform_request_callback(platform_request_cb cb) { (void)cb; } -STUB - "$clang" -target "$triple" -isysroot "$sdk_path" -fPIC \ - -c "$build_dir/ocaml_stub.c" -o "$build_dir/journal_complete_stub.o" - ocaml_object="$build_dir/journal_complete_stub.o" -fi - -# --- deterministic link-input staging (mirrors the lui script) ------------ -fingerprint=$(shasum -a 256 "$ocaml_object" "$build_dir/journal_lui_bridge.o" \ - | shasum -a 256 | cut -d ' ' -f 1) -link_dir="$build_dir/native-link-inputs/$fingerprint" -mkdir -p "$link_dir" -cp -f "$ocaml_object" "$link_dir/journal_complete.o" - -extra_inputs="" -if [[ -n ${JOURNAL_EXTRA_OBJECTS:-} ]]; then - extra_inputs=":${JOURNAL_EXTRA_OBJECTS}" -fi - -# --- SwiftPM -------------------------------------------------------------- -swift_args=( - build - --package-path "$swift_dir" - --product JournalApp -) -if [[ $platform == ios-simulator ]]; then - swift_args+=(--triple "$triple" --sdk "$sdk_path") - # Simulator binaries exec on the host macOS kernel, so entitlements baked - # into the code signature are validated as macOS entitlements and the exec - # is killed (error 163) no matter what identity signed them. The sim reads - # its entitlements from the __TEXT,__entitlements section instead — embed - # them at link time like Xcode does, then sign adhoc. This is what makes - # keychain (OAuth sign-in, localAccount) work on the sim. - ios_entitlements="$build_dir/ios-sim-entitlements.plist" - cp "$entitlements_dir/ios-debug-profile.entitlements" "$ios_entitlements" - bundle_id=$(plutil -extract CFBundleIdentifier raw "$info_plist") - ios_team_id=${JOURNAL_IOS_TEAM_ID:-K378MFWK59} - plutil -replace keychain-access-groups -json \ - "[\"$ios_team_id.$bundle_id\"]" "$ios_entitlements" - plutil -insert application-identifier -string \ - "$ios_team_id.$bundle_id" "$ios_entitlements" - swift_args+=(-Xlinker -sectcreate -Xlinker __TEXT -Xlinker __entitlements - -Xlinker "$ios_entitlements") -fi - -JOURNAL_LUI_PACKAGE_PATH=${JOURNAL_LUI_PACKAGE_PATH:-$repo_root/../lui/platform/apple} \ -JOURNAL_NATIVE_LINK_INPUTS="$link_dir/journal_complete.o$extra_inputs" \ -swift "${swift_args[@]}" - -# --- .app assembly --------------------------------------------------------- -product_dir="$swift_dir/.build/$triple/debug" -[[ -f $product_dir/JournalApp ]] || product_dir="$swift_dir/.build/debug" -app_dir=${app_dir_arg:-$build_dir/LogseqJournal.app} -rm -rf "$app_dir" -register_callback() { - python3 - "$1" <<'PYPLIST' -import plistlib, sys -from pathlib import Path -path = Path(sys.argv[1]) -info = plistlib.loads(path.read_bytes()) -entries = info.setdefault('CFBundleURLTypes', []) -if not any('logseqjournal' in entry.get('CFBundleURLSchemes', []) for entry in entries): - entries.append({'CFBundleURLName': 'com.logseq.journal.auth', 'CFBundleURLSchemes': ['logseqjournal']}) -path.write_bytes(plistlib.dumps(info)) -PYPLIST -} -if [[ $platform == macos ]]; then - mkdir -p "$app_dir/Contents/MacOS" "$app_dir/Contents/Resources" - cp "$info_plist" "$app_dir/Contents/Info.plist" - register_callback "$app_dir/Contents/Info.plist" - cp "$product_dir/JournalApp" "$app_dir/Contents/MacOS/JournalApp" - bash "$repo_root/tool/package_journal_apple_icons.sh" "$platform" "$app_dir" - # keychain-access-groups needs a real team id; without one the group is - # invalid and AMFI kills the binary, so drop the key for local builds. With a - # team id, sign with the Apple Development identity so the entitlement is - # honored (OAuth/keychain then work on macOS too). - macos_entitlements="$build_dir/macos-entitlements.plist" - cp "$entitlements_dir/macos-debug-profile.entitlements" "$macos_entitlements" - bundle_id=$(plutil -extract CFBundleIdentifier raw "$info_plist") - if [[ -n ${JOURNAL_MACOS_TEAM_ID:-} ]]; then - plutil -replace keychain-access-groups -json \ - "[\"$JOURNAL_MACOS_TEAM_ID.$bundle_id\"]" "$macos_entitlements" - # macOS rejects the iOS-style `application-identifier` entitlement key; - # the application id is implied by the signature + keychain-access-groups. - plutil -remove application-identifier "$macos_entitlements" 2>/dev/null || true - codesign --force --sign "${JOURNAL_MACOS_SIGN_IDENTITY:-Apple Development}" \ - --timestamp=none --entitlements "$macos_entitlements" "$app_dir" || true - else - plutil -remove keychain-access-groups "$macos_entitlements" - codesign --force --sign - --timestamp=none \ - --entitlements "$macos_entitlements" \ - "$app_dir" || true - fi -else - # iOS bundles are flat; an empty Contents/ dir breaks install + codesign. - mkdir -p "$app_dir" - cp "$info_plist" "$app_dir/Info.plist" - register_callback "$app_dir/Info.plist" - cp "$product_dir/JournalApp" "$app_dir/JournalApp" - bash "$repo_root/tool/package_journal_apple_icons.sh" "$platform" "$app_dir" - # Plain adhoc signature — the sim's entitlements already live in the - # __TEXT,__entitlements section embedded at link time above. Do NOT pass - # --entitlements here: a signature-level entitlements blob is validated by - # the host kernel as macOS entitlements and the exec is killed. - codesign --force --sign - --timestamp=none "$app_dir" || true -fi - -echo "$app_dir" +exec python3 "$repo_root/tool/journal_build.py" "$@" diff --git a/tool/journal_build.py b/tool/journal_build.py new file mode 100644 index 00000000..5d745cc0 --- /dev/null +++ b/tool/journal_build.py @@ -0,0 +1,796 @@ +#!/usr/bin/env python3 +"""Implementation of build_journal_apple.sh. Python standard library only.""" +import argparse +from contextlib import contextmanager, ExitStack +from datetime import datetime, timezone +import fcntl +import fnmatch +import hashlib +import json +import os +from pathlib import Path +import plistlib +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_evidence(record, directory): + path = (directory / record["path"]).resolve() + if sha(path) != record["sha256"]: + raise ValueError(f"producer evidence hash mismatch: {path}") + return json.loads(path.read_text()) + + +def sdk_identity(sdk, platform, version): + path = Path(sdk) / "SDKSettings.plist" + settings = plistlib.loads(path.read_bytes()) + if settings["DefaultProperties"]["PLATFORM_NAME"] != platform or settings["Version"] != version: + raise ValueError(f"SDK platform/version differs from selected {platform} {version}: {sdk}") + return {"settings": {str(path): sha(path)}, "canonical_name": settings["CanonicalName"]} + + +def files(directory, native_only=False): + """Hash input bytes, including untracked edits, without traversing caches.""" + ignored = {".git", ".build", "_build", "node_modules", "__pycache__"} + result = {} + for parent, dirs, names in os.walk(directory): + dirs[:] = sorted(d for d in dirs if d not in ignored and not d.startswith(".ci-")) + for name in sorted(names): + path = Path(parent) / name + native = path.suffix in {".a", ".cmxa", ".cmx", ".cmi", ".cma", ".o", ".so", ".dylib", ".h"} or name in {"META", "dune-package"} + if path.is_file() and not name.endswith(".pyc") and (not native_only or native): + result[str(path.relative_to(directory))] = sha(path) + return result + + +def source_inputs(): + result = {} + for path in sorted(ROOT.iterdir()): + if path.name in {"app", "swift", "apple", "assets", "config", "tool"} or path.name.startswith("logseq_"): + if path.is_dir(): + result.update({str(path.relative_to(ROOT) / name): digest + for name, digest in files(path).items()}) + elif path.is_file(): + result[path.name] = sha(path) + elif path.name in {"dune-project", "dune", "dune-workspace"}: + result[path.name] = sha(path) + return result + + +@contextmanager +def resource_lock(path): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+") as stream: + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + stream.seek(0) + raise RuntimeError(f"busy: {path}: {stream.read().strip()}") from None + stream.seek(0) + stream.truncate() + stream.write(f"pid={os.getpid()} cwd={ROOT}\n") + stream.flush() + try: + yield + finally: + fcntl.flock(stream, fcntl.LOCK_UN) + + +class Build: + def __init__(self, args, run_dir): + self.args = args + self.run_dir = run_dir + self.phase = "preflight" + self.env = dict(os.environ) + # Xcode's Python launcher inserts SDK/include/library lookup variables. + # They must not affect the explicit target SDK or switch in subprocesses. + ignored = {name: self.env.pop(name) for name in ("SDKROOT", "CPATH", "LIBRARY_PATH", + "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH") if name in self.env} + mode = "experimental-host-restamp" if args.experimental_host_restamp else "link-only" if args.link_only else "native" + self.report = {"status": "running", "started": datetime.now(timezone.utc).isoformat(), + "commands": [], "mode": mode, "cross_abi_validated": False, + "runtime_acceptance": False, "installed": False, "ignored_environment": ignored} + + def save(self): + (self.run_dir / "manifest.json").write_text(json.dumps(self.report, indent=2) + "\n") + + def run(self, *argv, cwd=ROOT, env=None): + argv = [str(a) for a in argv] + record = {"phase": self.phase, "argv": argv, "cwd": str(cwd), + "started": datetime.now(timezone.utc).isoformat()} + self.report["commands"].append(record) + log = self.run_dir / (self.phase + ".log") + print(f"[{self.phase}] {shlex.join(argv)}", file=sys.stderr, flush=True) + # Stream output directly to the phase log, including while a compiler is running. + with log.open("a+") as stream: + stream.write("+ " + shlex.join(argv) + "\n") + stream.flush() + offset = stream.tell() + process = subprocess.Popen(argv, cwd=cwd, env=env or self.env, text=True, + stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + try: + code = process.wait() + except BaseException: + # Keep locks until this command's descendants have stopped. + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=5) + except ProcessLookupError: + pass + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise + stream.seek(offset) + output = stream.read() + record.update(exit_code=code, finished=datetime.now(timezone.utc).isoformat()) + self.save() + if code: + raise RuntimeError(f"{self.phase}: exit {code}; see {log}\n{output[-3000:]}") + return output.strip() + + def opam(self, *argv): + return self.run("opam", argv[0], "--switch", self.args.switch, *argv[1:]) + + def dependencies(self): + rows = [line.split(maxsplit=2) for line in self.opam( + "list", "--installed", "--columns=name,installed-version,pin", "--short", "--color=never").splitlines()] + installed = {row[0]: row[1] for row in rows} + current_pins = {row[0]: row[2] for row in rows if len(row) == 3} + wanted, pins = {}, {} + for lock in sorted(ROOT.glob("*.opam.locked")): + text = lock.read_text() + for package, version in re.findall(r'"([^"\s]+)"\s*\{=\s*"([^"\s]+)"\}', text.split("depends: [", 1)[1]): + if package in wanted and wanted[package] != version: + raise RuntimeError(f"conflicting locked versions: {package}") + wanted[package] = version + for package, url in re.findall(r'\[\s*"([^"]+)"\s*"(git\+[^"]+)"\s*\]', text): + if package in pins and pins[package] != url: + raise RuntimeError(f"conflicting locked pins: {package}") + pins[package] = url + local = {p.stem for p in ROOT.glob("*.opam")} + errors = [f"{name}: lock={version}, installed={installed.get(name, 'missing')}" + for name, version in wanted.items() if name not in local and installed.get(name) != version] + resolved = {} + # Both URL pins and CI's resolved local pins must resolve to a clean Git checkout. + ci_path = ROOT / ".ci-reports/dependencies.json" + ci = {r["package"]: r for r in json.loads(ci_path.read_text())} if ci_path.exists() else {} + for package, expected in pins.items(): + name, version = package.split(".", 1) + if installed.get(name) != version: + continue + metadata = self.prefix / ".opam-switch/packages" / package / "opam" + actual = self.opam("show", "--just-file", "--field=url.src", metadata).strip('"') + if current_pins.get(name) != actual: + errors.append(f"{name}: active pin {current_pins.get(name)!r} differs from installed source {actual!r}") + checkout = self.prefix / ".opam-switch/sources" / name + ref = expected.rsplit("#", 1)[1] + if actual != expected: + record = ci.get(package, {}) + checkout = (ROOT / record.get("checkout", "missing-ci-checkout")).resolve() + # A local pin needs the existing CI resolution record, including exact source URL. + if record.get("source") != expected or actual.removeprefix("file://") != str(checkout): + errors.append(f"{name}: installed pin {actual!r} differs from lock {expected!r}") + continue + ref = record.get("resolved_sha", "") + if not (checkout / ".git").exists(): + errors.append(f"{name}: resolved source checkout missing: {checkout}") + continue + try: + head = self.run("git", "-C", checkout, "rev-parse", "HEAD") + dirty = self.run("git", "-C", checkout, "status", "--porcelain", "--untracked-files=no") + except RuntimeError as error: + errors.append(f"{name}: invalid resolved source checkout: {error}") + continue + if dirty or (re.fullmatch(r"[0-9a-f]{40}", ref) and head != ref): + errors.append(f"{name}: dirty or wrong resolved source: {head}; expected {ref}") + resolved[package] = {"source": expected, "resolved_sha": head, "checkout": str(checkout)} + self.report["dependency_errors"] = errors + self.report["dependencies"] = {"installed": installed, "pins": resolved, + "freshness": "offline: moving refs use the currently installed resolved checkout; no remote update"} + if errors: + raise RuntimeError("switch/lock mismatch (no pins changed):\n" + "\n".join(errors)) + # Include installed native archives, stubs, interfaces and META in ABI identity. + self.report["dependencies"]["installed_lib_hashes"] = files(self.prefix / "lib", native_only=True) + return self.report["dependencies"] + + def imported_native(self, source): + """Read producer records as data; never execute their recorded commands.""" + envelope = json.loads(self.args.native_manifest.read_text()) + if envelope["schema_version"] != 1: + raise ValueError("unsupported native manifest schema_version") + directory = self.args.native_manifest.resolve().parent + pre, link, xmlm = [read_evidence(envelope[name], directory) + for name in ("source", "link", "xmlm_force_link")] + if pre["lui"] != self.lui_sha: + raise ValueError("producer LUI SHA differs from this checkout") + # Build/tool/docs commits may advance HEAD; native bytes/configuration below must still match exactly. + self.run("git", "merge-base", "--is-ancestor", pre["commit"], self.report["source_sha"]) + removed = self.run("git", "diff", "--name-only", "--diff-filter=D", pre["commit"], "--").splitlines() + for name in sorted(set(source) | set(removed)): + if (Path(name).name in {"dune", "dune-project", "dune-workspace"} + or name.endswith((".opam", ".opam.locked")) or name == "tool/native_static_link_flags.sh"): + original = subprocess.check_output(["git", "show", pre["commit"] + ":" + name], cwd=ROOT) + if hashlib.sha256(original).hexdigest() != source.get(name): + raise ValueError(f"native build configuration changed since producer commit: {name}") + lock = (ROOT / "logseq_journal.opam.locked").read_text() + for key, package in (("signal", "ocaml-signal"), ("datascript", "datascript-ocaml-native")): + expected = re.search(r'"' + package + r'\.[^"]+"\s*"git\+[^"#]+#([0-9a-f]{40})"', lock).group(1) + if pre[key] != expected: + raise ValueError(f"producer {package} differs from locked commit") + native_sources = dict(pre["native_source_sha256"]) + for name, digest in link["device_c_source_sha256"].items(): + native_sources["logseq_sync/lib/effect_runner/platform/" + name] = digest + required = {name: digest for name, digest in source.items() + if Path(name).suffix in {".ml", ".mli", ".c", ".h"} + and not {"tool", "test", "apple-tests"}.intersection(Path(name).parts)} + mismatch = [name for name in sorted(set(required) | set(native_sources)) + if native_sources.get(name) != required.get(name)] + if mismatch: + raise ValueError("producer native sources do not cover/match: " + ", ".join(mismatch)) + compiler = Path(link["command"][0]) + host = Path(pre["production_native_object"]) + if pre["production_native_object_sha256"] != link["production_host_sha256"]: + raise ValueError("producer host object records disagree") + self.native_include = compiler.parent.parent / "lib/ocaml" + for name in ("mldoc", "xmlm"): + expected = re.search(r'"' + name + r'\.dev"\s*"git\+[^"#]+#([0-9a-f]{40})"', lock).group(1) + if link["new_dependency_provenance"][name]["pinned_commit"] != expected: + raise ValueError(f"producer {name} differs from locked commit") + if "validated_build" in envelope: + inputs, historical = self.validated_native(envelope, directory, pre, link) + else: + inputs = self.producer_files(pre, link, xmlm) + historical = {} + inputs.update({str(host): link["production_host_sha256"], + str(self.args.native_manifest.resolve()): sha(self.args.native_manifest)}) + for record in envelope.values(): + if isinstance(record, dict) and "path" in record: + inputs[str((directory / record["path"]).resolve())] = record["sha256"] + historical = {p: h for p, h in historical.items() if p not in inputs} + for path, digest in inputs.items(): + if sha(path) != digest: + raise ValueError(f"native input hash mismatch: {path}") + self.verify_target(host, "MACOS") + symbols = self.run("nm", "-j", "-U", host) + if not all(name in symbols for name in ("_caml_startup", "_camlMldoc", "_camlXmlm")): + raise ValueError("host complete object must retain runtime, mldoc and XMLm") + if any(name in symbols for name in ("_logseq_transport_dns_start", "_logseq_journal_crypto_call")): + raise ValueError("host object still provides DNS/crypto; target C must provide them") + self.host_object, self.native_files = host, inputs + self.report["native_source_sha"] = pre["commit"] + self.report["abi_limit"] = "EXPERIMENTAL: host ARM64 OCaml/runtime restamped; full iOS runtime ABI is not validated" + print(self.report["abi_limit"], file=sys.stderr) + return {"producer": envelope, "files": inputs, "historical_inputs": historical, + "compiler_version": link["compiler_version"], "producer_command": link["command"], + "source_sha": pre["commit"], "pins": {key: pre[key] for key in ("lui", "signal", "datascript")}, + "xmlm": "single-unit force-link; original cmx/object bytes verified"} + + def validated_native(self, envelope, directory, pre, link): + """Reuse a fixed complete object, with a hash-bound prior validation receipt.""" + receipt = read_evidence(envelope["validated_build"], directory) + previous = receipt["inputs"]["dependencies"] + if (receipt["status"] != "built" or receipt["mode"] != "experimental-host-restamp" + or previous.get("source_sha", receipt["source_sha"]) != pre["commit"] + or not receipt["production_link"]["runtime_mldoc_xmlm_retained"] + or previous["compiler_version"] != link["compiler_version"] + or previous["xmlm"] != "single-unit force-link; original cmx/object bytes verified"): + raise ValueError("validated build receipt does not prove this producer object") + for name in ("source", "link", "xmlm_force_link"): + if previous["producer"][name] != envelope[name]: + raise ValueError(f"validated build receipt producer evidence differs: {name}") + verified = {**previous.get("historical_inputs", {}), **previous["files"]} + expected = {**link["direct_native_input_sha256"], + pre["production_native_object"]: link["production_host_sha256"], + link["command"][0]: link["compiler_sha256"]} + for name in ("source", "link", "xmlm_force_link"): + record = envelope[name] + expected[str((directory / record["path"]).resolve())] = record["sha256"] + if any(verified.get(path) != digest for path, digest in expected.items()): + raise ValueError("validated build receipt native object/input hashes differ from producer") + # These headers are still consumed by this build's fresh target C compilation. + headers = {str(p): sha(p) for p in self.native_include.rglob("*.h")} + recorded = {p: h for p, h in verified.items() + if p.endswith(".h") and self.native_include in Path(p).parents} + if not recorded or headers != recorded: + raise ValueError("current OCaml runtime headers differ from the validated producer ABI") + self.report["producer_validation"] = {"receipt": envelope["validated_build"], + "consumed_inputs": "historical hash proof only; old archive paths are not live build inputs"} + return headers, verified + + def producer_files(self, pre, link, xmlm): + """First import verifies still-present producer inputs and XMLm bytes in full.""" + inputs = dict(link["direct_native_input_sha256"]) + compiler = Path(link["command"][0]) + if sha(compiler) != link["compiler_sha256"]: + raise ValueError("producer compiler hash mismatch") + inputs[str(compiler)] = link["compiler_sha256"] + inputs.update(pre["installed_lui_source_sha256"]) + for name in ("mldoc", "xmlm"): + dependency = link["new_dependency_provenance"][name] + archive = Path(dependency["installed_archive"]) + inputs[str(archive)] = dependency["archive_sha256"] + inputs[str(archive.with_suffix(".a"))] = dependency["static_archive_sha256"] + for filename, record in dependency["installed_sources_matched_to_pinned_git"].items(): + inputs[str(archive.parent / filename)] = record["sha256"] + # Only XMLm's original unit receives force-link metadata. Never broad -linkall. + command = xmlm["command"] + if len(command) != 6 or command[:4] != [str(compiler), "-a", "-linkall", "-o"]: + raise ValueError("XMLm proof must describe a single-unit -a -linkall archive") + forced, cmx = Path(command[4]), Path(command[5]) + original = Path(link["new_dependency_provenance"]["xmlm"]["installed_archive"]) + if forced.name != "xmlm.cmxa" or cmx.name != "xmlm.cmx" or str(forced) not in link["command"]: + raise ValueError("XMLm force-link archive was not a native link input") + if sha(original.parent / "xmlm.cmx") != xmlm["original_cmx_sha256"] or sha(cmx) != xmlm["original_cmx_sha256"]: + raise ValueError("XMLm original and force-linked cmx bytes differ") + for archive in (original.with_suffix(".a"), forced.with_suffix(".a")): + member = subprocess.check_output(["ar", "-p", str(archive), "xmlm.o"]) + if hashlib.sha256(member).hexdigest() != xmlm["original_object_sha256"]: + raise ValueError("XMLm force-link changed original object bytes") + info_tool = compiler.with_name("ocamlobjinfo") + if "Force link: YES" not in self.run(info_tool, forced): + raise ValueError("XMLm archive lacks force-link metadata") + inputs.update({str(cmx): xmlm["original_cmx_sha256"], str(original.parent / "xmlm.cmx"): xmlm["original_cmx_sha256"], + str(info_tool): sha(info_tool)}) + # The producer removed only host DNS/crypto from a local archive copy. + effect = next(Path(p).parent for p in link["command"] if p.endswith("/logseq_sync_effect_runner_impl.cmxa")) + for name in ("logseq_sync_effect_runner_impl.cmxa", "logseq_sync_effect_runner_impl.a"): + original_path = next(p for p in inputs if Path(p).name == name) + inputs[str(effect / name)] = inputs[original_path] + patched = effect / "liblogseq_sync_effect_runner_impl_stubs.a" + inputs[str(patched)] = link["patched_host_stub_archive_sha256"] + if "platform_crypto_stubs.o" in self.run("ar", "-t", patched): + raise ValueError("producer host DNS/crypto C member was not excluded") + inputs.update({str(p): sha(p) for p in self.native_include.rglob("*.h")}) + if self.run(compiler, "-version") != link["compiler_version"]: + raise ValueError("producer OCaml compiler version differs") + return inputs + + def preflight(self): + a = self.args + if self.run("uname", "-m") != "arm64": + raise RuntimeError("only Darwin arm64 is supported") + sdk_name = {"macos": "macosx", "ios-simulator": "iphonesimulator", "ios-device": "iphoneos"}[a.platform] + sdk = self.run("xcrun", "--sdk", sdk_name, "--show-sdk-path") + triple = f"arm64-apple-{'macosx' if a.platform == 'macos' else 'ios'}{a.deployment_target}" + if a.platform == "ios-simulator": + triple += "-simulator" + self.sdk, self.triple = sdk, triple + # The existing Dune/icon helpers read these legacy variables. Bind them to this CLI's target. + self.env.update(JOURNAL_APPLE_SDK_ROOT=sdk, JOURNAL_MACOS_DEPLOYMENT_TARGET=a.deployment_target, + JOURNAL_IOS_DEPLOYMENT_TARGET=a.deployment_target) + self.swift_dir = Path(os.environ.get("JOURNAL_SWIFT_DIR", ROOT / "swift")).resolve() + self.plist = Path(os.environ.get("JOURNAL_INFO_PLIST", ROOT / "apple/Info.plist")).resolve() + self.entitlements = Path(os.environ.get("JOURNAL_ENTITLEMENTS_DIR", ROOT / "config/entitlements")).resolve() + self.lui = Path(a.lui_package or os.environ.get("JOURNAL_LUI_PACKAGE_PATH", ROOT / "../lui/platform/apple")).resolve() + if not (self.lui / "Package.swift").is_file(): + raise RuntimeError(f"LUI Apple package missing: {self.lui}; set --lui-package") + # Production Swift LUI must agree with the OCaml lock; link-only may intentionally inspect another backend. + lui_sha = self.lui_sha = self.run("git", "-C", self.lui, "rev-parse", "HEAD") + if not a.link_only: + expected = re.search(r'"git\+https://github.com/logseq/lui.git#([0-9a-f]{40})"', + (ROOT / "logseq_journal.opam.locked").read_text()).group(1) + if lui_sha != expected or self.run("git", "-C", self.lui, "status", "--porcelain", "--untracked-files=no"): + raise RuntimeError(f"LUI Swift checkout must be clean at locked {expected}, found {lui_sha}") + self.info = plistlib.loads(self.plist.read_bytes()) + for key in ("CFBundleIdentifier", "CFBundleExecutable", "CFBundleShortVersionString", "CFBundleVersion"): + if not self.info.get(key): + raise RuntimeError(f"missing {key}: {self.plist}") + if self.info["CFBundleExecutable"] != "JournalApp": + raise RuntimeError("CFBundleExecutable must be JournalApp") + if a.version: + self.info["CFBundleShortVersionString"] = a.version + if a.build_number: + self.info["CFBundleVersion"] = a.build_number + for key in ("CFBundleShortVersionString", "CFBundleVersion"): + if not re.fullmatch(r"[0-9]+(?:\.[0-9]+){0,2}", self.info[key]): + raise RuntimeError(f"invalid numeric {key}: {self.info[key]}") + self.report["source_sha"] = self.run("git", "rev-parse", "HEAD") + self.report["source_status"] = self.run("git", "status", "--porcelain") + tools = {name: self.run("xcrun", "--sdk", sdk_name, "--find", name) + for name in ("swift", "clang", "actool")} + toolchain = {"paths": tools, "hashes": {k: sha(v) for k, v in tools.items()}, + "swift": self.run(tools["swift"], "--version"), "xcode": self.run("xcodebuild", "-version"), + "sdk": sdk, "sdk_version": self.run("xcrun", "--sdk", sdk_name, "--show-sdk-version"), + "path": self.env.get("PATH"), "developer_dir": self.env.get("DEVELOPER_DIR")} + toolchain["sdk_identity"] = sdk_identity(sdk, sdk_name, toolchain["sdk_version"]) + self.tools = tools + source = source_inputs() + dependencies = {"mode": "stub; OCaml dependencies not used or validated"} + if a.native_manifest: + dependencies = self.imported_native(source) + elif not a.link_only: + self.prefix = Path(self.opam("var", "prefix")) + dependencies = self.dependencies() + toolchain["ocaml"] = self.opam("exec", "--", "ocamlopt", "-config") + toolchain["compiler_sha256"] = sha(self.prefix / "bin/ocamlopt.opt") + if "architecture: arm64" not in toolchain["ocaml"] or "system: macosx" not in toolchain["ocaml"]: + raise RuntimeError("macOS build requires native arm64 macosx OCaml compiler") + toolchain["dune"] = self.opam("exec", "--", "dune", "--version") + gmp = Path(self.run("pkg-config", "--variable=libdir", "gmp")) / "libgmp.a" + toolchain["gmp"] = {"path": str(gmp), "sha256": sha(gmp)} + signing = self.device_signing() if a.platform == "ios-device" else {} + self.inputs = {"source": source, "swift": files(self.swift_dir), "lui": files(self.lui), + "lui_sha": lui_sha, "plist": sha(self.plist), "entitlements": files(self.entitlements), + "dependencies": dependencies, "toolchain": toolchain, "target": triple, + "signing": signing, "flags": {"configuration": a.configuration, "mode": self.report["mode"], + "info": self.info, "sign_identity": a.sign_identity}} + self.inputs["flags"]["teams"] = {name: self.env.get(name) for name in ("JOURNAL_IOS_TEAM_ID", "JOURNAL_MACOS_TEAM_ID")} + self.report.update(inputs=self.inputs, target=triple, sdk_name=sdk_name, architecture="arm64", + cache_key=fingerprint(self.inputs)) + self.save() + + def compile(self, workspace): + a = self.args + self.phase = "native" + if a.link_only: + source = workspace / "ocaml_stub.c" + source.write_text(STUB) + native = workspace / "journal_stub.o" + self.run(self.tools["clang"], "-target", self.triple, "-isysroot", self.sdk, + "-fPIC", "-c", source, "-o", native) + elif a.native_manifest: + native = workspace / "journal_complete_restamped.o" + platform = "2" if a.platform == "ios-device" else "7" + self.run("xcrun", "vtool", "-set-build-version", platform, a.deployment_target, + self.inputs["toolchain"]["sdk_version"], "-replace", "-output", native, self.host_object) + self.target_c = workspace / "platform_crypto.o" + self.run(self.tools["clang"], "-target", self.triple, "-isysroot", self.sdk, + "-O2", "-fno-strict-aliasing", "-fwrapv", "-pthread", "-D_FILE_OFFSET_BITS=64", "-Wall", "-g", + "-I", self.native_include, "-c", ROOT / "logseq_sync/lib/effect_runner/platform/platform_crypto_stubs.c", + "-o", self.target_c) + self.verify_target(self.target_c) + else: + dune_dir = workspace / "dune" + self.env.update(DUNE_CACHE_ROOT=str(workspace / "dune-cache"), + XDG_CACHE_HOME=str(workspace / "xdg-cache")) + self.opam("exec", "--", "dune", "build", "--root", ROOT, "--no-config", "--cache=disabled", + "--build-dir", dune_dir, + "--profile", "dev" if a.configuration == "debug" else "release", + "-j", str(a.jobs), "app/native_embed.exe.o") + native = dune_dir / "default/app/native_embed.exe.o" + symbols = self.run("nm", "-U", native) + for library in ("Mldoc", "Xmlm"): + if not re.search(r"caml" + library + r"(?:__|_)", symbols): + raise RuntimeError(f"native object does not retain {library}; linker rule needs review. " + "Do not reuse a historical link argv or edit protected Dune without approval.") + self.report["native_object"] = {"path": str(native), "sha256": sha(native), "mode": self.report["mode"]} + self.phase = "swift" + scratch = workspace / "swift" + native_inputs = [native, self.target_c] if a.native_manifest else [native] + self.env.update(JOURNAL_LUI_PACKAGE_PATH=str(self.lui), JOURNAL_NATIVE_LINK_INPUTS=":".join(map(str, native_inputs)), + CLANG_MODULE_CACHE_PATH=str(workspace / "clang-modules"), + SWIFTPM_MODULECACHE_OVERRIDE=str(workspace / "swift-modules")) + argv = [self.tools["swift"], "build", "--package-path", self.swift_dir, "--scratch-path", scratch, + "--cache-path", workspace / "swift-cache", "--config-path", workspace / "swift-config", + "--security-path", workspace / "swift-security", "--product", "JournalApp", + "--configuration", a.configuration, "--triple", self.triple, "--sdk", self.sdk, + "--skip-update", "--disable-automatic-resolution", "--disable-index-store", "-j", str(a.jobs), + "-Xlinker", "-syslibroot", "-Xlinker", self.sdk] + if a.platform == "ios-simulator": + ent = plistlib.loads((self.entitlements / "ios-debug-profile.entitlements").read_bytes()) + bundle = self.info["CFBundleIdentifier"] + team = os.environ.get("JOURNAL_IOS_TEAM_ID", "K378MFWK59") + ent.update({"keychain-access-groups": [f"{team}.{bundle}"], "application-identifier": f"{team}.{bundle}"}) + ent_path = workspace / "sim-entitlements.plist" + ent_path.write_bytes(plistlib.dumps(ent)) + argv += ["-Xlinker", "-sectcreate", "-Xlinker", "__TEXT", "-Xlinker", "__entitlements", + "-Xlinker", ent_path] + if a.native_manifest: + self.link_map = workspace / "native-link-map.txt" + argv += ["-Xlinker", "-map", "-Xlinker", self.link_map] + self.run(*argv) + # Query the same product/configuration; never fall back to another platform's .build/debug. + product = Path(self.run(*argv, "--show-bin-path").splitlines()[-1]) / "JournalApp" + self.verify_target(product) + if a.native_manifest: + self.verify_production_link(product) + return product + + def verify_target(self, product, platform=None): + if self.run("xcrun", "lipo", "-archs", product) != "arm64": + raise RuntimeError(f"wrong binary architecture: {product}") + output = self.run("xcrun", "vtool", "-show-build", product) + platform = platform or {"macos": "MACOS", "ios-simulator": "IOSSIMULATOR", "ios-device": "IOS"}[self.args.platform] + if not re.search(r"platform\s+" + platform + r"\b", output): + raise RuntimeError(f"wrong Mach-O platform, expected {platform}: {product}") + + def verify_production_link(self, product): + symbols = self.run("nm", "-j", "-U", product) + if not all(name in symbols for name in ("_caml_startup", "_camlMldoc", "_camlXmlm")): + raise ValueError("final product dropped OCaml runtime, mldoc or XMLm") + text = self.link_map.read_text() + objects = {int(m[1]): m[2].strip() for line in text.splitlines() + if (m := re.match(r"^\[\s*(\d+)\]\s+(.+)$", line))} + for symbol in ("_logseq_transport_dns_start", "_logseq_journal_crypto_call"): + rows = [line for line in text.splitlines() if line.split() and line.split()[-1] == symbol] + match = re.search(r"\[\s*(\d+)\]", rows[0]) if len(rows) == 1 else None + if not match or objects[int(match[1])] != str(self.target_c): + raise ValueError(f"final {symbol} provider is not the freshly compiled target C object") + swift_objects = {} + for line in (product.parent / "JournalApp.product/Objects.LinkFileList").read_text().splitlines(): + obj = Path(shlex.split(line)[0]) + self.verify_target(obj) + swift_objects[str(obj)] = sha(obj) + dylibs = self.run("otool", "-L", product).splitlines()[1:] + if any(not line.strip().startswith(("/System/Library/", "/usr/lib/")) for line in dylibs): + raise ValueError("product links a non-system dynamic library") + self.report["production_link"] = {"link_map": str(self.link_map), "link_map_sha256": sha(self.link_map), + "target_c": {"path": str(self.target_c), "sha256": sha(self.target_c)}, "swift_objects": swift_objects, + "runtime_mldoc_xmlm_retained": True, "cross_abi_validated": False} + + def device_signing(self): + a = self.args + profile = plistlib.loads(self.run("security", "cms", "-D", "-i", a.provisioning_profile).encode()) + if "iOS" not in profile.get("Platform", []): + raise ValueError("provisioning profile is not for iOS") + if profile["ExpirationDate"].replace(tzinfo=timezone.utc) <= datetime.now(timezone.utc): + raise ValueError("provisioning profile expired") + certificate = a.sign_identity.upper() + if certificate not in {hashlib.sha1(c).hexdigest().upper() for c in profile["DeveloperCertificates"]}: + raise ValueError("signing certificate is not in the provisioning profile") + if certificate not in self.run("security", "find-identity", "-v", "-p", "codesigning"): + raise ValueError("requested signing identity is unavailable") + self.device_entitlements = plistlib.loads(a.sign_entitlements.read_bytes()) + self.team = profile["TeamIdentifier"][0] + if self.device_entitlements.get("application-identifier") != f"{self.team}.{self.info['CFBundleIdentifier']}": + raise ValueError("device entitlement application-identifier does not match team/bundle") + def allowed(requested, grant): + if isinstance(requested, list): + return isinstance(grant, list) and all(any(allowed(x, y) for y in grant) for x in requested) + if isinstance(requested, str) and isinstance(grant, str): + return fnmatch.fnmatchcase(requested, grant) + return requested == grant + grants = profile["Entitlements"] + if any(key not in grants or not allowed(value, grants[key]) for key, value in self.device_entitlements.items()): + raise ValueError("requested device entitlements exceed provisioning profile") + self.report["device_signing"] = {"team": self.team, "certificate": certificate, "profile_uuid": profile["UUID"], + "expires": profile["ExpirationDate"].isoformat()} + return {str(p.resolve()): sha(p) for p in (a.provisioning_profile, a.sign_entitlements)} + + def assemble(self, product, app): + self.phase = "package" + # mkdir is exclusive: no existing output, rollback app or arbitrary directory is removed. + app.mkdir(parents=True, exist_ok=False) + mac = self.args.platform == "macos" + resource = app / "Contents/Resources" if mac else app + executable = app / "Contents/MacOS/JournalApp" if mac else app / "JournalApp" + plist = app / "Contents/Info.plist" if mac else app / "Info.plist" + resource.mkdir(parents=True, exist_ok=True) + executable.parent.mkdir(parents=True, exist_ok=True) + info = plistlib.loads(plistlib.dumps(self.info)) + info.update(JournalSourceRevision=self.report["source_sha"], + JournalBuildInputDigest=self.report["cache_key"], JournalBuildMode=self.report["mode"]) + if self.args.experimental_host_restamp: + info.update(JournalOCamlRuntimeOrigin="host-arm64-restamp", JournalCrossABIValidated=False) + info["LSMinimumSystemVersion" if mac else "MinimumOSVersion"] = self.args.deployment_target + info.setdefault("CFBundleURLTypes", []).append({"CFBundleURLName": "com.logseq.journal.auth", + "CFBundleURLSchemes": ["logseqjournal"]}) + plist.write_bytes(plistlib.dumps(info)) + shutil.copy2(product, executable) + # SwiftPM resource bundles must travel with the executable. + for bundle in product.parent.glob("*.bundle"): + shutil.copytree(bundle, resource / bundle.name) + self.run("bash", ROOT / "tool/package_journal_apple_icons.sh", self.args.platform, app) + if not (resource / "Assets.car").is_file(): + raise RuntimeError("icon Assets.car missing") + packaged = plistlib.loads(plist.read_bytes()) + if not any(key in packaged for key in ("CFBundleIconName", "CFBundleIcons")): + raise RuntimeError("packaged icon declaration missing") + self.phase = "sign" + command = ["codesign", "--force", "--sign", self.args.sign_identity, "--timestamp=none"] + team = None + if mac: + ent = plistlib.loads((self.entitlements / "macos-debug-profile.entitlements").read_bytes()) + # Local adhoc signing; a team identity requires the explicit team id. + ent.pop("application-identifier", None) + team = os.environ.get("JOURNAL_MACOS_TEAM_ID") + if self.args.sign_identity != "-": + if not team: + raise RuntimeError("JOURNAL_MACOS_TEAM_ID required for a development identity") + ent["keychain-access-groups"] = [f"{team}.{info['CFBundleIdentifier']}"] + else: + ent.pop("keychain-access-groups", None) + ent_path = self.run_dir / "sign-entitlements.plist" + ent_path.write_bytes(plistlib.dumps(ent)) + command += ["--entitlements", ent_path] + elif self.args.platform == "ios-device": + team = self.team + shutil.copy2(self.args.provisioning_profile, app / "embedded.mobileprovision") + command += ["--entitlements", self.args.sign_entitlements, "--generate-entitlement-der"] + self.run(*command, app) + self.run("codesign", "--verify", "--deep", "--strict", app) + signature = self.run("codesign", "-d", "--verbose=4", app) + if team and f"TeamIdentifier={team}" not in signature: + raise RuntimeError("signature team differs from requested team") + if self.args.platform == "ios-device": + output = self.run("codesign", "-d", "--entitlements", ":-", app) + signed = plistlib.loads(output[output.index("") + 8].encode()) + if signed != self.device_entitlements: + raise ValueError("signed entitlements differ from the requested device entitlements") + self.verify_target(executable) + self.report["artifact"] = {"app": str(app), "info": packaged, "files": files(app), "signature": signature, + "executable_sha256": sha(executable), "sign_identity": self.args.sign_identity} + + +def arguments(): + parser = argparse.ArgumentParser(description="Build Journal Apple host; never installs or updates dependencies.") + parser.add_argument("platform", choices=("macos", "ios-simulator", "ios-device"), nargs="?", default="macos") + parser.add_argument("app_directory", nargs="?", help="legacy positional output path; must not exist") + parser.add_argument("--app-dir", type=Path) + parser.add_argument("--switch", help="explicit pre-provisioned opam switch (native macOS only)") + parser.add_argument("--lui-package", type=Path, help="local LUI platform/apple package") + parser.add_argument("--preflight", action="store_true", help="check inputs without compiling") + parser.add_argument("--link-only", action="store_true", help="explicit stub build; not runnable acceptance") + parser.add_argument("--experimental-host-restamp", action="store_true", help="explicit existing host ARM64 compatibility route; full iOS ABI unverified") + parser.add_argument("--native-manifest", type=Path, help="hash-verified source/link/XMLm producer evidence envelope") + parser.add_argument("--cache-dir", type=Path, default=Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "logseq-journal-build") + parser.add_argument("--configuration", choices=("debug", "release"), default="debug") + parser.add_argument("--jobs", type=int, default=2) + parser.add_argument("--deployment-target", default="26.0") + parser.add_argument("--version") + parser.add_argument("--build-number") + parser.add_argument("--sign-identity", default="-", help="macOS identity; default adhoc, Simulator always adhoc") + parser.add_argument("--provisioning-profile", type=Path, help="existing device profile; never downloaded") + parser.add_argument("--sign-entitlements", type=Path, help="existing device signing entitlements") + a = parser.parse_args() + for name in ("JOURNAL_OCAML_OBJECT", "JOURNAL_EXTRA_OBJECTS", "JOURNAL_NATIVE_LINK_INPUTS"): + if os.environ.get(name): + parser.error(f"{name} is unproven input; use native macOS --switch or explicit --link-only. " + "Legacy Dune aliases/probe object injection need provenance support; host restamp is not cross compilation.") + if bool(a.native_manifest) != a.experimental_host_restamp: + parser.error("--native-manifest and --experimental-host-restamp must be supplied together") + if a.experimental_host_restamp and (a.link_only or a.switch or a.platform == "macos"): + parser.error("experimental import is an iOS mode, separate from --link-only and --switch") + if a.platform == "ios-device" and not a.experimental_host_restamp: + parser.error("iPhone build blocked: no verified complete OCaml iOS cross toolchain/ABI route; host restamp is unsupported") + if a.platform == "ios-simulator" and not (a.link_only or a.experimental_host_restamp): + parser.error("Simulator runtime build blocked; use --link-only for Swift/link checks. Host restamp is not verified cross compilation.") + if not (a.link_only or a.native_manifest or a.switch): + parser.error("native macOS requires --switch; no implicit switch/pin changes") + if a.platform == "ios-simulator" and a.sign_identity != "-": + parser.error("Simulator requires adhoc signature; entitlements live in __TEXT") + if a.platform == "ios-device" and (not a.provisioning_profile or not a.sign_entitlements or not re.fullmatch(r"[0-9A-Fa-f]{40}", a.sign_identity)): + parser.error("device build requires --provisioning-profile, --sign-entitlements and certificate SHA1 --sign-identity") + a.app_dir = a.app_dir or (Path(a.app_directory) if a.app_directory else None) + if a.app_dir and a.app_dir.exists(): + parser.error(f"output exists; preserve it and choose a new path: {a.app_dir}") + if a.jobs < 1 or not re.fullmatch(r"[0-9]+(?:\.[0-9]+){1,2}", a.deployment_target) or int(a.deployment_target.split(".")[0]) < 26: + parser.error("jobs must be positive and deployment-target must be numeric >= 26.0 (Swift package minimum)") + # Ambient compiler and lookup overrides invalidate the declared switch/SDK identity. + for name in ("OCAMLPATH", "OCAMLLIB", "OCAMLPARAM", "OCAMLFIND_CONF", "OCAMLFIND_TOOLCHAIN", "DUNE_PROFILE", + "DUNE_ROOT", "DUNE_WORKSPACE", "DUNE_CROSS_TARGET", "DUNE_INSTRUMENT_WITH", "DUNE_TARGET_EXEC", + "OCAMLFIND_COMMANDS", "OCAMLFIND_LDCONF", "CAML_LD_LIBRARY_PATH", "MACOSX_DEPLOYMENT_TARGET", + "IPHONEOS_DEPLOYMENT_TARGET", "ARCHFLAGS", "CFLAGS", "CPPFLAGS", "LDFLAGS", "CC", "CXX", "LD", "AR", + "SWIFT_EXEC", "PKG_CONFIG_PATH", "PKG_CONFIG_LIBDIR", "PKG_CONFIG_SYSROOT_DIR"): + if os.environ.get(name): + parser.error(f"unset ambient {name}; this entry point selects explicit tools/SDK and switch") + a.cache_dir = a.cache_dir.expanduser().resolve() + if a.cache_dir == ROOT or ROOT in a.cache_dir.parents: + parser.error("--cache-dir must be outside the repository") + return a + + +def main(): + def interrupted(signum, frame): + raise KeyboardInterrupt(f"signal {signum}") + signal.signal(signal.SIGTERM, interrupted) + a = arguments() + a.cache_dir.mkdir(parents=True, exist_ok=True) + with ExitStack() as stack: + stack.enter_context(resource_lock(a.cache_dir / "build.lock")) + run_dir = Path(tempfile.mkdtemp(prefix="run-", dir=a.cache_dir)) + build = Build(a, run_dir) + try: + if not a.link_only and not a.native_manifest: + prefix = Path(build.opam("var", "prefix")) + # opam uses POSIX record locks. Keep its switch read-locked for the whole build. + switch_lock = stack.enter_context((prefix / ".opam-switch/lock").open("rb")) + try: + fcntl.lockf(switch_lock, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError: + raise RuntimeError(f"busy: opam switch is being modified: {prefix}") from None + build.preflight() + if not a.preflight: + workspace = a.cache_dir / "objects" / build.report["cache_key"] + workspace.mkdir(parents=True, exist_ok=True) + build.report["cache_reused"] = (workspace / "inputs.json").exists() + (workspace / "inputs.json").write_text(json.dumps(build.inputs, indent=2) + "\n") + product = build.compile(workspace) + build.assemble(product, a.app_dir or run_dir / "LogseqJournal.app") + build.phase = "verify-inputs" + if (source_inputs() != build.inputs["source"] or files(build.swift_dir) != build.inputs["swift"] + or files(build.lui) != build.inputs["lui"] or sha(build.plist) != build.inputs["plist"] + or files(build.entitlements) != build.inputs["entitlements"]): + raise RuntimeError("source changed during build; artifact is not accepted, rerun with stable inputs") + if any(sha(path) != build.inputs["toolchain"]["hashes"][name] for name, path in build.tools.items()): + raise RuntimeError("Apple toolchain changed during build; artifact is not accepted") + if any(sha(p) != h for p, h in build.inputs["toolchain"]["sdk_identity"]["settings"].items()): + raise RuntimeError("SDK identity changed during build; artifact is not accepted") + imported = getattr(build, "native_files", {}) + if any(sha(p) != h for p, h in {**imported, **build.inputs["signing"]}.items()): + raise RuntimeError("imported native/signing inputs changed during build; artifact is not accepted") + if not a.link_only and not a.native_manifest and build.dependencies() != build.inputs["dependencies"]: + raise RuntimeError("switch changed during build; artifact is not accepted") + build.report["status"] = "preflight-passed" if a.preflight else "built" + build.report["finished"] = datetime.now(timezone.utc).isoformat() + build.save() + print(build.report.get("artifact", {}).get("app", "preflight passed")) + print(f"manifest: {run_dir / 'manifest.json'}", file=sys.stderr) + except (RuntimeError, OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError, KeyboardInterrupt) as error: + build.report.update(status="failed", failed_phase=build.phase, error=str(error)) + build.save() + print(f"build failed [{build.phase}]: {error}\nmanifest: {run_dir / 'manifest.json'}", file=sys.stderr) + return 1 + return 0 + + +STUB = """// Link-validation stub matching app/journal_lui_bridge.c's exported entries. +// Explicit --link-only mode; never runtime acceptance. +#include +typedef void (*patch_cb)(const char *); +typedef void (*wakeup_cb)(void); +typedef void (*platform_request_cb)(const char *, int32_t); +int32_t lui_ocaml_start(patch_cb cb, int32_t p, int32_t h, const char *d, int32_t l) + { (void)p; (void)h; (void)d; (void)l; if (cb) cb(""); return 1; } +int32_t lui_ocaml_stop(void) { return 1; } +int32_t lui_ocaml_appear(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_press(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_long_press(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_text_changed(int64_t n, const char *t) { (void)n; (void)t; return 1; } +int32_t lui_ocaml_submit(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_dismiss(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_double_press(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_toggle_changed(int64_t n, int32_t c) { (void)n; (void)c; return 1; } +int32_t lui_ocaml_radio_changed(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_slider_changed(int64_t n, double v) { (void)n; (void)v; return 1; } +int32_t lui_ocaml_press_detail(int64_t n, double x, double y, int32_t m, int32_t b, const char *t) + { (void)n; (void)x; (void)y; (void)m; (void)b; (void)t; return 1; } +int32_t lui_ocaml_pointer_down(int64_t n, double x, double y, int32_t m, int32_t b, const char *t) + { return lui_ocaml_press_detail(n, x, y, m, b, t); } +int32_t lui_ocaml_pointer_up(int64_t n, double x, double y, int32_t m, int32_t b, const char *t) + { return lui_ocaml_press_detail(n, x, y, m, b, t); } +int32_t lui_ocaml_context_menu_press(int64_t n, double x, double y, int32_t m, int32_t b, const char *t) + { return lui_ocaml_press_detail(n, x, y, m, b, t); } +int32_t lui_ocaml_pointer_enter(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_pointer_leave(int64_t n) { (void)n; return 1; } +int32_t lui_ocaml_scroll_completed(int64_t n, int64_t token, const char *outcome) + { (void)n; (void)token; (void)outcome; return 1; } +int32_t lui_ocaml_visible_range(int64_t n, int64_t first, int64_t last) + { (void)n; (void)first; (void)last; return 1; } +int32_t lui_ocaml_picked(int64_t n, const char *payload) { (void)n; (void)payload; return 1; } +int32_t journal_ocaml_extension_event(int64_t n, const char *name, const char *p) + { (void)n; (void)name; (void)p; return 1; } +int32_t journal_ocaml_pump(void) { return 1; } +void journal_ocaml_platform_event(const char *d, int32_t l) { (void)d; (void)l; } +void journal_ocaml_platform_response(const char *d, int32_t l) { (void)d; (void)l; } +void journal_ocaml_platform_failure(const char *d, int32_t l) { (void)d; (void)l; } +void journal_ocaml_set_wakeup_callback(wakeup_cb cb) { (void)cb; } +void journal_ocaml_set_platform_request_callback(platform_request_cb cb) { (void)cb; }""" + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (RuntimeError, OSError) as error: + print(f"build blocked: {error}", file=sys.stderr) + sys.exit(1) diff --git a/tool/package_journal_apple_icons.sh b/tool/package_journal_apple_icons.sh index a6c85016..8c76826d 100644 --- a/tool/package_journal_apple_icons.sh +++ b/tool/package_journal_apple_icons.sh @@ -3,7 +3,7 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -platform=${1:?usage: package_journal_apple_icons.sh APP_DIR} +platform=${1:?usage: package_journal_apple_icons.sh APP_DIR} app_dir=${2:?missing app directory} case "$platform" in macos) @@ -13,8 +13,9 @@ case "$platform" in info_plist="$app_dir/Contents/Info.plist" target_devices=(--target-device mac) ;; - ios-simulator) + ios-simulator|ios-device) sdk=iphonesimulator + [[ $platform != ios-device ]] || sdk=iphoneos deployment_target=${JOURNAL_IOS_DEPLOYMENT_TARGET:-26.0} resources="$app_dir" info_plist="$app_dir/Info.plist" diff --git a/tool/test_apple_app_icon.py b/tool/test_apple_app_icon.py index 884accdf..fb95240d 100644 --- a/tool/test_apple_app_icon.py +++ b/tool/test_apple_app_icon.py @@ -13,7 +13,7 @@ class AppIconPackagingTests(unittest.TestCase): def test_journal_icon_is_packaged(self): with tempfile.TemporaryDirectory(prefix="journal-app-icon-") as directory: - for platform in ("ios-simulator", "macos"): + for platform in ("ios-simulator", "ios-device", "macos"): with self.subTest(platform=platform): app = pathlib.Path(directory) / (platform + ".app") info_path = app / ("Contents/Info.plist" if platform == "macos" else "Info.plist") diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py new file mode 100644 index 00000000..c909dc24 --- /dev/null +++ b/tool/test_journal_build.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Small build-boundary checks; no device, dependency install or large build.""" +import importlib.util +from contextlib import contextmanager +from functools import partial +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "tool/build_journal_apple.sh" + + +class BuildBoundaryTests(unittest.TestCase): + def cli(self, *args, env=None): + return subprocess.run(["bash", str(SCRIPT), *args], text=True, + capture_output=True, env=env) + + def test_help_is_read_only_and_describes_modes(self): + result = self.cli("--help") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--link-only", result.stdout) + self.assertIn("--preflight", result.stdout) + self.assertIn("--experimental-host-restamp", result.stdout) + + def test_iphone_is_explicitly_blocked(self): + result = self.cli("ios-device", "--preflight") + self.assertNotEqual(result.returncode, 0) + self.assertIn("cross", result.stderr) + + def test_simulator_never_silently_uses_host_runtime(self): + result = self.cli("ios-simulator", "--preflight") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--link-only", result.stderr) + + def test_unproven_object_is_rejected_even_when_restamped(self): + env = dict(os.environ, JOURNAL_OCAML_OBJECT="/tmp/restamped-host.o") + result = self.cli("ios-simulator", "--link-only", "--preflight", env=env) + self.assertNotEqual(result.returncode, 0) + self.assertIn("JOURNAL_OCAML_OBJECT", result.stderr) + + def test_no_implicit_switch_or_environment_mutation(self): + result = self.cli("macos", "--preflight") + self.assertNotEqual(result.returncode, 0) + self.assertIn("--switch", result.stderr) + + def test_native_import_requires_both_explicit_mode_and_manifest(self): + for args, expected in [ + (("ios-device", "--native-manifest", "/tmp/input.json"), "--experimental-host-restamp"), + (("ios-device", "--experimental-host-restamp"), "--native-manifest"), + ]: + with self.subTest(args=args): + result = self.cli(*args, "--preflight") + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + + def test_foreign_dune_root_and_runtime_cannot_change_provenance(self): + for name in ("DUNE_ROOT", "DUNE_WORKSPACE", "DUNE_CROSS_TARGET", "DUNE_INSTRUMENT_WITH", "OCAMLLIB", + "MACOSX_DEPLOYMENT_TARGET", "CAML_LD_LIBRARY_PATH", "OCAMLFIND_COMMANDS"): + with self.subTest(name=name): + result = self.cli("macos", "--switch", "not-used", "--preflight", + env=dict(os.environ, **{name: "/tmp/foreign-build-input"})) + self.assertNotEqual(result.returncode, 0) + self.assertIn(name, result.stderr) + + def test_existing_output_is_preserved_before_toolchain_checks(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "Keep.app" + output.mkdir() + sentinel = output / "sentinel" + sentinel.write_text("keep") + result = self.cli("macos", "--link-only", "--app-dir", str(output)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("exists", result.stderr) + self.assertEqual(sentinel.read_text(), "keep") + + +class CacheBoundaryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = ROOT / "tool/journal_build.py" + if not path.exists(): + return + spec = importlib.util.spec_from_file_location("journal_build", path) + cls.build = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.build) + + def test_source_toolchain_dependency_flags_and_target_separate_caches(self): + self.assertTrue(hasattr(self, "build"), "build input fingerprint missing") + baseline = dict(source="a", dependencies="b", toolchain="c", flags=[], target="macos") + key = self.build.fingerprint(baseline) + self.assertEqual(key, self.build.fingerprint(dict(reversed(list(baseline.items()))))) + for field in baseline: + changed = dict(baseline, **{field: "changed"}) + self.assertNotEqual(key, self.build.fingerprint(changed), field) + + def test_second_process_cannot_take_live_lock(self): + self.assertTrue(hasattr(self, "build"), "cross-process build lock missing") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "resource.lock" + with self.build.resource_lock(path): + code = ("import sys;sys.path.insert(0,sys.argv[1]);" + "from journal_build import resource_lock;from pathlib import Path;" + "\nwith resource_lock(Path(sys.argv[2])): pass") + result = subprocess.run(["python3", "-c", code, str(ROOT / "tool"), str(path)], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("busy", result.stderr) + with self.build.resource_lock(path): + pass + + def test_link_stub_defines_current_swift_runtime_entries(self): + self.assertTrue(hasattr(self, "build"), "link stub missing") + required = set() + for path in (ROOT / "swift").glob("*.swift"): + required.update(re.findall(r'@_silgen_name\("((?:lui|journal)_ocaml_[^"]+)"\)', path.read_text())) + with tempfile.TemporaryDirectory() as directory: + source, obj = Path(directory) / "stub.c", Path(directory) / "stub.o" + source.write_text(self.build.STUB) + subprocess.run(["xcrun", "clang", "-c", str(source), "-o", str(obj)], check=True, + capture_output=True) + defined = subprocess.check_output(["nm", "-U", str(obj)], text=True) + missing = sorted(name for name in required if "_" + name not in defined) + self.assertEqual(missing, [], "stub missing current Swift bridge entries") + + def test_producer_evidence_hash_is_checked_before_reading(self): + self.assertTrue(hasattr(self.build, "read_evidence"), "verified producer evidence reader missing") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "producer.json" + path.write_text('{"source_sha":"first"}') + record = {"path": path.name, "sha256": self.build.sha(path)} + self.assertEqual(self.build.read_evidence(record, Path(directory))["source_sha"], "first") + path.write_text('{"source_sha":"changed"}') + with self.assertRaisesRegex(ValueError, "hash"): + self.build.read_evidence(record, Path(directory)) + + def test_simulator_object_cannot_enter_device_packaging(self): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + source, obj = directory / "target.c", directory / "sim.o" + source.write_text("int target_marker = 1;\n") + subprocess.run(["xcrun", "clang", "-target", "arm64-apple-ios26.0-simulator", + "-c", str(source), "-o", str(obj)], check=True, capture_output=True) + build = self.build.Build(SimpleNamespace(link_only=False, experimental_host_restamp=True, + platform="ios-device"), directory) + with self.assertRaisesRegex(RuntimeError, "expected IOS"): + build.verify_target(obj) + + @contextmanager + def producer_fixture(self): + """A tiny real Mach-O producer; all mutable files live outside Git.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + package = root / "opam" + compiler = package / "bin/ocamlopt.opt" + header = package / "lib/ocaml/caml/mlvalues.h" + header.parent.mkdir(parents=True) + header.write_text("/* original runtime ABI header */\n") + native = root / "host.o" + c = root / "host.c" + c.write_text("void caml_startup(void) {}\nint camlMldoc; int camlXmlm;\n") + subprocess.run(["xcrun", "clang", "-target", "arm64-apple-macosx26.0", "-c", c, "-o", native], + check=True, capture_output=True) + repo = root / "repo" + (repo / "app").mkdir(parents=True) + platform = repo / "logseq_sync/lib/effect_runner/platform" + platform.mkdir(parents=True) + (repo / "app/unit.ml").write_text("let value = 1\n") + for name in ("platform_crypto_stubs.c", "platform_crypto_stubs.h"): + (platform / name).write_text("/* target source */\n") + pins = {"lui": "1" * 40, "signal": "2" * 40, "datascript": "3" * 40, + "mldoc": "4" * 40, "xmlm": "5" * 40} + names = {"signal": "ocaml-signal", "datascript": "datascript-ocaml-native"} + (repo / "logseq_journal.opam.locked").write_text("\n".join( + f'"{names.get(k, k)}.dev" "git+https://example.invalid/{k}#{v}"' for k, v in pins.items())) + subprocess.run(["git", "init", "-q", repo], check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "add", "."], check=True, capture_output=True) + subprocess.run(["git", "-C", repo, "-c", "user.name=Build test", "-c", "user.email=build@example.invalid", + "commit", "-qm", "producer"], check=True, capture_output=True) + commit = subprocess.check_output(["git", "-C", repo, "rev-parse", "HEAD"], text=True).strip() + with patch.object(self.build, "ROOT", repo): + source = self.build.source_inputs() + host_hash = self.build.sha(native) + history = root / "consumed.cmxa" + history.write_text("original archive") + forced = root / "xmlm-force/xmlm.cmxa" + cmx = root / "xmlm-force/xmlm.cmx" + effect = root / "effect/logseq_sync_effect_runner_impl.cmxa" + dummy = "a" * 64 + inputs = {str(history): self.build.sha(history), str(compiler): dummy, str(forced): dummy, + str(forced.with_suffix('.a')): dummy, str(effect): dummy, + str(effect.with_suffix('.a')): dummy, str(native): host_hash, + str(header): self.build.sha(header)} + pre = dict(commit=commit, lui=pins['lui'], signal=pins['signal'], datascript=pins['datascript'], + production_native_object=str(native), production_native_object_sha256=host_hash, + native_source_sha256={k: v for k, v in source.items() if k.endswith('.ml')}, + installed_lui_source_sha256={}) + link = dict(command=[str(compiler), str(effect), str(forced)], compiler_sha256=dummy, + compiler_version="5.3.0", production_host_sha256=host_hash, + direct_native_input_sha256={k: v for k, v in inputs.items() if k not in (str(native), str(header))}, + device_c_source_sha256={p.name: self.build.sha(p) for p in platform.iterdir()}, + patched_host_stub_archive_sha256=dummy, + new_dependency_provenance={k: dict(pinned_commit=pins[k], + installed_archive=str(package / f"lib/{k}/{k}.cmxa"), archive_sha256=dummy, + static_archive_sha256=dummy, installed_sources_matched_to_pinned_git={}) + for k in ('mldoc', 'xmlm')}) + xmlm = dict(command=[str(compiler), "-a", "-linkall", "-o", str(forced), str(cmx)], + original_cmx_sha256=dummy, local_cmx_sha256=dummy, + original_object_sha256=dummy, object_bytes_identical=True) + envelope = {'schema_version': 1} + for name, value in [('source', pre), ('link', link), ('xmlm_force_link', xmlm)]: + path = root / (name + '.json') + path.write_text(json.dumps(value)) + envelope[name] = dict(path=str(path), sha256=self.build.sha(path)) + inputs[str(path)] = self.build.sha(path) + for name in ('mldoc', 'xmlm'): + archive = Path(link['new_dependency_provenance'][name]['installed_archive']) + inputs.update({str(archive): dummy, str(archive.with_suffix('.a')): dummy}) + inputs.update({str(cmx): dummy, str(package / 'lib/xmlm/xmlm.cmx'): dummy, + str(effect.parent / 'liblogseq_sync_effect_runner_impl_stubs.a'): dummy}) + receipt = dict(status='built', mode='experimental-host-restamp', source_sha=commit, + production_link=dict(runtime_mldoc_xmlm_retained=True, cross_abi_validated=False), + inputs=dict(source=source, dependencies=dict(producer=dict(envelope), files=inputs, + compiler_version='5.3.0', xmlm='single-unit force-link; original cmx/object bytes verified'))) + receipt_path = root / 'receipt.json' + receipt_path.write_text(json.dumps(receipt)) + envelope['validated_build'] = dict(path=str(receipt_path), sha256=self.build.sha(receipt_path)) + path = root / 'input.json' + path.write_text(json.dumps(envelope)) + build = self.build.Build(SimpleNamespace(native_manifest=path, experimental_host_restamp=True, + link_only=False, platform='ios-device'), root) + build.run = partial(build.run, cwd=repo) + build.report['source_sha'], build.lui_sha = commit, pins['lui'] + yield SimpleNamespace(build=build, source=source, history=history, native=native, header=header, + envelope=envelope, path=path, receipt=receipt, receipt_path=receipt_path, repo=repo) + + def test_consumed_archives_may_change_or_disappear_with_fixed_object_receipt(self): + with self.producer_fixture() as f: + f.history.write_text("later unrelated compilation") + imported = f.build.imported_native(f.source) + self.assertNotIn(str(f.history), imported['files']) + self.assertIn(str(f.history), imported['historical_inputs']) + f.history.unlink() + f.build.imported_native(f.source) + + def test_builder_only_commit_keeps_exact_native_source_identity(self): + with self.producer_fixture() as f: + (f.repo / 'tool').mkdir() + (f.repo / 'tool/builder.txt').write_text('new builder revision') + subprocess.run(['git', '-C', f.repo, 'add', '.'], check=True, capture_output=True) + subprocess.run(['git', '-C', f.repo, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'builder only'], check=True, capture_output=True) + f.build.report['source_sha'] = subprocess.check_output( + ['git', '-C', f.repo, 'rev-parse', 'HEAD'], text=True).strip() + imported = f.build.imported_native(self.build.source_inputs()) + self.assertEqual(imported['source_sha'], f.receipt['source_sha']) + f.source['app/unit.ml'] = 'f' * 64 + with self.assertRaisesRegex(ValueError, 'native sources'): + f.build.imported_native(f.source) + + def test_fixed_object_source_and_current_c_headers_remain_strict(self): + for case in ('object', 'source_sha', 'native_source', 'target_c', 'target_header', 'runtime_header'): + with self.subTest(case=case), self.producer_fixture() as f: + if case == 'object': + f.native.write_bytes(f.native.read_bytes() + b'changed') + elif case == 'source_sha': + f.build.report['source_sha'] = 'f' * 40 + elif case == 'runtime_header': + f.header.write_text('changed ABI header') + else: + key = {'native_source': 'app/unit.ml', 'target_c': 'logseq_sync/lib/effect_runner/platform/platform_crypto_stubs.c', + 'target_header': 'logseq_sync/lib/effect_runner/platform/platform_crypto_stubs.h'}[case] + f.source[key] = 'f' * 64 + with self.assertRaises((ValueError, RuntimeError)): + f.build.imported_native(f.source) + + def test_producer_and_receipt_mismatches_cannot_authorize_an_object(self): + for case in ('producer_bytes', 'receipt_bytes', 'receipt_source', 'receipt_object', 'receipt_evidence'): + with self.subTest(case=case), self.producer_fixture() as f: + if case == 'producer_bytes': + Path(f.envelope['link']['path']).write_text('{}') + elif case == 'receipt_bytes': + f.receipt_path.write_text('{}') + else: + if case == 'receipt_source': + f.receipt['source_sha'] = 'f' * 40 + elif case == 'receipt_object': + f.receipt['inputs']['dependencies']['files'][str(f.native)] = 'f' * 64 + else: + f.receipt['inputs']['dependencies']['producer']['source']['sha256'] = 'f' * 64 + f.receipt_path.write_text(json.dumps(f.receipt)) + f.envelope['validated_build']['sha256'] = self.build.sha(f.receipt_path) + f.path.write_text(json.dumps(f.envelope)) + with self.assertRaises((ValueError, RuntimeError)): + f.build.imported_native(f.source) + + def test_device_sdk_cannot_be_replaced_by_simulator_sdk(self): + sdk = Path(subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', '--show-sdk-path'], text=True).strip()) + version = subprocess.check_output(['xcrun', '--sdk', 'iphonesimulator', '--show-sdk-version'], text=True).strip() + self.assertTrue(hasattr(self.build, 'sdk_identity'), 'SDK identity validator missing') + with self.assertRaisesRegex(ValueError, 'SDK'): + self.build.sdk_identity(sdk, 'iphoneos', version) + + +if __name__ == "__main__": + unittest.main() From a678928399858bb745de441ada7cf55953359655 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 11:06:25 +0800 Subject: [PATCH 2/7] Stop remaining build process group before releasing cache lock --- .../2026-10-15-reusable-apple-build.md | 3 +- tool/journal_build.py | 10 ++-- tool/test_journal_build.py | 54 +++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index 42aabcc7..48131119 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -32,7 +32,8 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route ## Consequences - The existing shell entry point delegates to one standard-library Python implementation. The documented entry point replaces legacy object-injecting aliases for supported builds; protected Dune/spec/lock files are unchanged. -- Eighteen boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. +- Nineteen boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. +- Independent exact-head review reproduced a cancellation race: a leader could exit on SIGTERM while its descendant ignored it, allowing the cache lock to release early. Cancellation now kills the remaining process group even after the leader exits. A real descendant holding a FIFO write endpoint verifies termination before lock release; it failed before the fix and passes afterward. - An actual experimental iPhoneOS build produced a signed production App: runtime/mldoc/XMLm retained, 55 Swift objects verified as iOS ARM64, DNS/crypto link-map providers verified as freshly compiled iPhoneOS C, and profile/signature/entitlements read back successfully. Subsequent review tightened source deletion/configuration, compiler and profile guards. A final retry correctly stopped before compilation when another authorized task changed 14 historical producer archive inputs; it did not accept the old provenance or interfere with that cache. The successful App remains preserved. Independent journal-code-review found no remaining substantive issue after fixes. - The later archive mismatch exposed an overly strict importer lifecycle rule: the fixed complete object remained valid while another owner normally updated its consumed archives. Receipt-bound historical provenance now distinguishes that state from changed current inputs. After the repair, a final production iPhoneOS build passed through this entry point: fixed OCaml complete object reused, target C and all Swift/LUI objects compiled in a new cache namespace, icons/resources packaged, signature/entitlements and input stability verified. No OCaml recompilation or phone operation occurred. - Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. diff --git a/tool/journal_build.py b/tool/journal_build.py index 5d745cc0..8c0438d5 100644 --- a/tool/journal_build.py +++ b/tool/journal_build.py @@ -134,10 +134,14 @@ def run(self, *argv, cwd=ROOT, env=None): try: os.killpg(process.pid, signal.SIGTERM) process.wait(timeout=5) - except ProcessLookupError: + except (ProcessLookupError, subprocess.TimeoutExpired): pass - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) + finally: + # The leader may exit before a descendant that ignores SIGTERM. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass process.wait() raise stream.seek(offset) diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py index c909dc24..bc5b4ba2 100644 --- a/tool/test_journal_build.py +++ b/tool/test_journal_build.py @@ -9,6 +9,10 @@ import re import subprocess import tempfile +import signal +import sys +import threading +import time from types import SimpleNamespace import unittest from unittest.mock import patch @@ -115,6 +119,56 @@ def test_second_process_cannot_take_live_lock(self): with self.build.resource_lock(path): pass + def test_cancellation_stops_descendant_even_when_parent_exits_first(self): + with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + ready = directory / 'child.pid' + pipe = directory / 'writer-lifetime' + os.mkfifo(pipe) + reader = os.open(pipe, os.O_RDONLY | os.O_NONBLOCK) + stop = threading.Event() + code = '''import os,signal,sys,time +from pathlib import Path +if os.fork() == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + fd = os.open(sys.argv[2], os.O_WRONLY) + Path(sys.argv[1]).write_text(str(os.getpid())) + time.sleep(10) + os._exit(0) +while True: time.sleep(1) +''' + def cancel_when_ready(): + for _ in range(300): + if ready.exists(): + os.kill(os.getpid(), signal.SIGINT) + return + if stop.wait(.01): + return + sender = threading.Thread(target=cancel_when_ready, daemon=True) + build = self.build.Build(SimpleNamespace(link_only=True, experimental_host_restamp=False), directory) + try: + with self.build.resource_lock(directory / 'cache.lock'): + sender.start() + with self.assertRaises(KeyboardInterrupt): + build.run(sys.executable, '-c', code, ready, pipe) + closed = False + for _ in range(100): + try: + closed = os.read(reader, 1) == b'' + break + except BlockingIOError: + time.sleep(.01) + self.assertTrue(closed, 'cancelled descendant still holds writer before lock release') + finally: + os.close(reader) + stop.set() + sender.join(timeout=1) + if ready.exists(): + try: + os.kill(int(ready.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + def test_link_stub_defines_current_swift_runtime_entries(self): self.assertTrue(hasattr(self, "build"), "link stub missing") required = set() From 9bdcc6df6cff004dd5cbe0ed5fad402c95bb9df9 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 11:14:37 +0800 Subject: [PATCH 3/7] Reject untracked Swift sources in pinned LUI checkout --- .../2026-10-15-reusable-apple-build.md | 3 ++- tool/journal_build.py | 2 +- tool/test_journal_build.py | 27 ++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index 48131119..ef9dd2e2 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -32,8 +32,9 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route ## Consequences - The existing shell entry point delegates to one standard-library Python implementation. The documented entry point replaces legacy object-injecting aliases for supported builds; protected Dune/spec/lock files are unchanged. -- Nineteen boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. +- Twenty boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. - Independent exact-head review reproduced a cancellation race: a leader could exit on SIGTERM while its descendant ignored it, allowing the cache lock to release early. Cancellation now kills the remaining process group even after the leader exits. A real descendant holding a FIFO write endpoint verifies termination before lock release; it failed before the fix and passes afterward. +- The final independent review also reproduced untracked Swift source being included by SwiftPM while the LUI pin guard ignored it. Production LUI checks now include all untracked files; a preflight regression rejects the added Swift source before native import. The successful production builds used a checkout without this trigger. - An actual experimental iPhoneOS build produced a signed production App: runtime/mldoc/XMLm retained, 55 Swift objects verified as iOS ARM64, DNS/crypto link-map providers verified as freshly compiled iPhoneOS C, and profile/signature/entitlements read back successfully. Subsequent review tightened source deletion/configuration, compiler and profile guards. A final retry correctly stopped before compilation when another authorized task changed 14 historical producer archive inputs; it did not accept the old provenance or interfere with that cache. The successful App remains preserved. Independent journal-code-review found no remaining substantive issue after fixes. - The later archive mismatch exposed an overly strict importer lifecycle rule: the fixed complete object remained valid while another owner normally updated its consumed archives. Receipt-bound historical provenance now distinguishes that state from changed current inputs. After the repair, a final production iPhoneOS build passed through this entry point: fixed OCaml complete object reused, target C and all Swift/LUI objects compiled in a new cache namespace, icons/resources packaged, signature/entitlements and input stability verified. No OCaml recompilation or phone operation occurred. - Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. diff --git a/tool/journal_build.py b/tool/journal_build.py index 8c0438d5..1a731dfb 100644 --- a/tool/journal_build.py +++ b/tool/journal_build.py @@ -393,7 +393,7 @@ def preflight(self): if not a.link_only: expected = re.search(r'"git\+https://github.com/logseq/lui.git#([0-9a-f]{40})"', (ROOT / "logseq_journal.opam.locked").read_text()).group(1) - if lui_sha != expected or self.run("git", "-C", self.lui, "status", "--porcelain", "--untracked-files=no"): + if lui_sha != expected or self.run("git", "-C", self.lui, "status", "--porcelain", "--untracked-files=all"): raise RuntimeError(f"LUI Swift checkout must be clean at locked {expected}, found {lui_sha}") self.info = plistlib.loads(self.plist.read_bytes()) for key in ("CFBundleIdentifier", "CFBundleExecutable", "CFBundleShortVersionString", "CFBundleVersion"): diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py index bc5b4ba2..7fdbc4b6 100644 --- a/tool/test_journal_build.py +++ b/tool/test_journal_build.py @@ -5,6 +5,7 @@ from functools import partial import json import os +import plistlib from pathlib import Path import re import subprocess @@ -139,7 +140,7 @@ def test_cancellation_stops_descendant_even_when_parent_exits_first(self): ''' def cancel_when_ready(): for _ in range(300): - if ready.exists(): + if ready.exists() and ready.read_text(): os.kill(os.getpid(), signal.SIGINT) return if stop.wait(.01): @@ -318,6 +319,30 @@ def test_builder_only_commit_keeps_exact_native_source_identity(self): with self.assertRaisesRegex(ValueError, 'native sources'): f.build.imported_native(f.source) + def test_production_lui_pin_rejects_untracked_swift_sources(self): + with self.producer_fixture() as f: + lui = f.repo.parent / 'lui' + target = lui / 'Sources/Backend' + target.mkdir(parents=True) + (lui / 'Package.swift').write_text('// swift-tools-version: 6.0\nimport PackageDescription\n' + 'let package = Package(name: "Fixture", targets: [.target(name: "Backend")])\n') + (target / 'Tracked.swift').write_text('func tracked() {}\n') + subprocess.run(['git', 'init', '-q', lui], check=True, capture_output=True) + subprocess.run(['git', '-C', lui, 'add', '.'], check=True, capture_output=True) + subprocess.run(['git', '-C', lui, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'pinned LUI'], check=True, capture_output=True) + pinned = subprocess.check_output(['git', '-C', lui, 'rev-parse', 'HEAD'], text=True).strip() + (target / 'Unpinned.swift').write_text('func unpinned() {}\n') + (f.repo / 'logseq_journal.opam.locked').write_text(f'"git+https://github.com/logseq/lui.git#{pinned}"') + (f.repo / 'apple').mkdir() + (f.repo / 'apple/Info.plist').write_bytes(plistlib.dumps(dict(CFBundleIdentifier='fixture.journal', + CFBundleExecutable='JournalApp', CFBundleVersion='1.0', CFBundleShortVersionString='1.0'))) + f.build.args.lui_package, f.build.args.deployment_target = lui, '26.0' + f.build.args.version = f.build.args.build_number = None + with patch.object(f.build, 'imported_native', side_effect=AssertionError('unpinned LUI reached native import')): + with self.assertRaisesRegex(RuntimeError, 'LUI Swift checkout must be clean'): + f.build.preflight() + def test_fixed_object_source_and_current_c_headers_remain_strict(self): for case in ('object', 'source_sha', 'native_source', 'target_c', 'target_header', 'runtime_header'): with self.subTest(case=case), self.producer_fixture() as f: From a44b90eafc8cbdf1ff0aefe9795e532ea5cfb3dc Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 11:32:23 +0800 Subject: [PATCH 4/7] Move detailed Journal build guidance into repository skill --- .agents/skills/journal-build/SKILL.md | 68 +++++++++++++++++++ AGENTS.md | 49 +------------ .../2026-10-15-reusable-apple-build.md | 1 + 3 files changed, 70 insertions(+), 48 deletions(-) create mode 100644 .agents/skills/journal-build/SKILL.md diff --git a/.agents/skills/journal-build/SKILL.md b/.agents/skills/journal-build/SKILL.md new file mode 100644 index 00000000..fd2a629c --- /dev/null +++ b/.agents/skills/journal-build/SKILL.md @@ -0,0 +1,68 @@ +--- +name: journal-build +description: Build and verify Journal Apple apps for macOS, iOS Simulator and iPhoneOS using the repository entry point and explicit native provenance. +--- + +# Journal Build + +Run from the repository root with `bash tool/build_journal_apple.sh`; use `--help` for current options. It shares the existing Dune, Swift and icon helpers. Build never installs, touches a graph, updates dependencies/pins or deletes caches. Coordinate CPU/actool and borrowed inputs with any active build owner. + +## Select inputs and target + +All targets are arm64 with deployment target >=26.0. Choose an external `--cache-dir` and local LUI `platform/apple` package. Production LUI must be clean at its locked Git pin, including untracked files. + +| Target | SDK | Mode / prerequisites | +| --- | --- | --- | +| `macos` | `macosx` | Native Dune object; explicit prepared `--switch` matching locks/pins. Adhoc signing by default. | +| `ios-simulator` | `iphonesimulator` | Explicit `--link-only` stub for Swift/link/package checks, or the experimental mode below. | +| `ios-device` | `iphoneos` | Explicit `--experimental-host-restamp --native-manifest FILE`, existing certificate SHA1/profile/entitlements. | + +Default iOS runtime builds and raw `JOURNAL_OCAML_OBJECT`/extra-object injection are refused. Host ARM64 OCaml/runtime restamping is an experimental compatibility route, **not verified full iOS cross-compilation or ABI acceptance**. Real target C/Swift and signing checks do not establish full OCaml ABI or startup acceptance. + +Simulator signing is adhoc. macOS development signing needs `--sign-identity` plus `JOURNAL_MACOS_TEAM_ID`; device mode requires the existing signing inputs shown below. + +```sh +bash tool/build_journal_apple.sh macos --switch YOUR_SWITCH \ + --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache --preflight +bash tool/build_journal_apple.sh ios-simulator --link-only \ + --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache +bash tool/build_journal_apple.sh ios-device --experimental-host-restamp \ + --native-manifest ../native-input.json --lui-package /path/to/lui/platform/apple \ + --cache-dir ../journal-build-cache --sign-identity CERTIFICATE_SHA1 \ + --provisioning-profile /path/to/device.mobileprovision \ + --sign-entitlements /path/to/device.entitlements --build-number 1.2.3 +``` + +`--preflight` checks without compilation; remove it for the macOS build. Unset ambient compiler/OCaml/SDK lookup overrides; SDK selection is explicit. Provision dependencies separately with the existing `tool/prepare_ci_dependencies.py` lock/resolved-pin workflow, which fetches and pins; use only an authorized switch. The builder never repairs it implicitly. Offline moving refs have no remote freshness claim. + +## Experimental object provenance + +The envelope references original producer records by SHA-256; paths are relative to the envelope, while producer records retain their own explicit paths: + +```json +{ + "schema_version": 1, + "source": {"path": "preflight.json", "sha256": "SHA256"}, + "link": {"path": "native-command-manifest.json", "sha256": "SHA256"}, + "xmlm_force_link": {"path": "xmlm-force-link-proof.json", "sha256": "SHA256"}, + "validated_build": {"path": "successful-build/manifest.json", "sha256": "SHA256"} +} +``` + +`source` binds native source hashes, producer commit, LUI/Signal/DataScript pins and complete-object hash. `link` binds compiler/dependency/archive identity, including mldoc/XMLm pins. `xmlm_force_link` proves a single XMLm unit retained its original cmx/object bytes. Recorded commands are data, never command hooks. + +`validated_build` is optional: use a trusted successful experimental manifest from this entry point binding the same records, native source, compiler and object. Without it, the original producer files and XMLm proof must still verify in full. With it, consumed archives/compiler/source paths become historical hash proof and may later change or disappear. This preserves object provenance; it does not prove those paths can still rebuild it. Keep original records/object/receipt read-only; never rewrite proofs or invent a receipt to bypass mismatch. + +The producer commit must be an ancestor of the builder revision. The full native source set, Dune/opam/link configuration and pins must still match exactly; Build/tool/docs commits may advance HEAD, changed native bytes need new evidence. Current object/platform/symbols, runtime headers used by fresh target C, records/receipt, SDK and Swift/signing inputs remain live checks. Reports distinguish builder `source_sha` and `native_source_sha`. + +## Cache and failure handling + +Identity includes source, actual dependency ABI inputs, toolchain/SDK, flags and target. Identical inputs reuse incremental state; changed inputs select another namespace. Every run leaves external phase logs and `run-*/manifest.json`. The cache has a nonblocking lock; native Dune builds also hold the switch read lock. A busy lock belongs to its active owner: coordinate or wait. Preserve existing output/rollback apps and choose a new `--app-dir`; nothing is cleared automatically. + +For failure, read `failed_phase`, `error` and that phase's log. Correct target/environment or select matching prepared inputs. Source/configuration/pin/current-input mismatch requires matching provenance; a historical archive change alone can use the validated receipt contract above. Missing mldoc/XMLm retention requires linker-rule review; protected Dune/spec/lock edits still follow the repository's authorization rules. Legacy object-injecting Dune aliases/probe callers need migration to this entry point. + +## Accept and hand off + +Only `status: built` confirms an artifact. Check recorded revisions/dirty hashes, target/SDK/flags, bundle/executable hashes, version and signing identity. Resources/icons precede signing; device profile/certificate/team/entitlements and strict signature are checked. Experimental builds additionally check Swift object platforms, fresh target C link-map providers and runtime/mldoc/XMLm retention. Listing a `.cmxa` alone does not establish retention. + +`preflight-passed` and `link-only` are not runtime acceptance. Preserve `cross_abi_validated=false` for host-restamped objects and state which compilation was reused. Carry the manifest with the App; installation and phone/graph operations are separate from build. For builder changes, use `PYTHONDONTWRITEBYTECODE=1 python3 tool/test_journal_build.py` and the existing icon test as appropriate. diff --git a/AGENTS.md b/AGENTS.md index 8b14dbd3..fb25b82e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,51 +12,4 @@ ## Build -Use `bash tool/build_journal_apple.sh --help` as the Apple build entry point. It uses the existing Dune native object, Swift package and icon packager; it never installs an app, touches a graph, changes pins, downloads dependencies, or clears caches. Provision dependencies separately in an explicitly chosen switch using the existing CI lock/resolved-pin workflow (`tool/prepare_ci_dependencies.py`); that workflow fetches and pins dependencies and must not be run against another task's shared switch without authorization. - -| Target | SDK / architecture | Current support | -| --- | --- | --- | -| `macos` | `macosx` / arm64 | Native OCaml build through an explicit matching switch; default signing is adhoc. | -| `ios-simulator --link-only` | `iphonesimulator` / arm64 | Explicit stub: Swift/link/package/signature checks only; no runnable OCaml acceptance. | -| iOS `--experimental-host-restamp --native-manifest FILE` | selected `iphonesimulator` or `iphoneos` / arm64 | Explicit compatibility mode: verified producer host OCaml object, freshly compiled target DNS/crypto C and Swift. Full OCaml/runtime iOS ABI remains unvalidated. | -| iOS without either explicit mode | distinct Simulator / iPhoneOS ABI | Refused; no implicit stub or host restamp. | - -Run from this worktree; choose an external cache and a local LUI `platform/apple` checkout. For native builds its clean Git HEAD must equal the LUI pin in `logseq_journal.opam.locked`. - -```sh -bash tool/build_journal_apple.sh macos --switch YOUR_SWITCH \ - --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache --preflight -bash tool/build_journal_apple.sh macos --switch YOUR_SWITCH \ - --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache -bash tool/build_journal_apple.sh ios-simulator --link-only \ - --lui-package /path/to/lui/platform/apple --cache-dir ../journal-build-cache -bash tool/build_journal_apple.sh ios-device --experimental-host-restamp \ - --native-manifest ../native-input.json --lui-package /path/to/lui/platform/apple \ - --cache-dir ../journal-build-cache --sign-identity CERTIFICATE_SHA1 \ - --provisioning-profile /path/to/device.mobileprovision \ - --sign-entitlements /path/to/device.entitlements --build-number 1.2.3 -python3 tool/test_journal_build.py -``` - -The experimental input is a small envelope referencing existing producer records by SHA-256. Paths below are relative to the envelope; producer records retain their explicit input paths. Keep these records and the fixed complete object read-only. - -```json -{ - "schema_version": 1, - "source": {"path": "preflight.json", "sha256": "SHA256"}, - "link": {"path": "native-command-manifest.json", "sha256": "SHA256"}, - "xmlm_force_link": {"path": "xmlm-force-link-proof.json", "sha256": "SHA256"}, - "validated_build": {"path": "successful-build/manifest.json", "sha256": "SHA256"} -} -``` - -`source` binds commit/LUI/Signal/DataScript, production source hashes and the host complete-object hash. `link` supplies compiler/version, archive/stub input hashes and pinned mldoc/XMLm provenance. `xmlm_force_link` describes only `ocamlopt -a -linkall -o OWN/xmlm.cmxa OWN/xmlm.cmx`, with unchanged original cmx/object hashes. The producer commit must be an ancestor of the current builder revision, and the complete native source set plus Dune/opam/link configuration must still match its bytes exactly. Thus Build/tool/docs commits can advance HEAD; native changes cannot silently reuse an older object. Reports keep both current `source_sha` and `native_source_sha`. The importer never executes recorded commands. - -`validated_build` is optional and must reference a trusted successful experimental build manifest from this entry point, with the same original three producer records, native source, compiler identity and complete-object hash. Without it, a first import verifies all still-present producer files and XMLm bytes/metadata. With it, already-consumed archives/compiler/source paths are historical provenance, so subsequent builds may change or remove them without invalidating the fixed object. Their original hashes remain recorded; this does not prove that the original object can still be rebuilt from those paths. Current object bytes/platform/symbols, exact native source/configuration/pins, runtime headers used by fresh target C, evidence/receipt bytes, SDK and Swift/signing inputs remain checked. Never edit an original producer record or invent a successful receipt to bypass a mismatch. Changed native bytes require matching new producer evidence. - -- `--preflight` validates without compiling. The Dune route compares the explicit switch with locks/pins; the imported route validates producer source/dependency/compiler hashes instead of requiring its private dependencies to be globally installed. Neither repairs pins or installs dependencies. Moving refs have no network freshness claim. -- Every run creates an external `run-*/manifest.json` and phase logs. Only `status: built` confirms a completed artifact; `preflight-passed` and `link-only` do not establish runtime behavior. The manifest records exact source SHA plus dirty input hashes, dependency/toolchain/SDK/target/flags, native input and final bundle hashes, signing and version. Keep it with the app when handing off for installation. -- Source, actual dependency native ABI inputs, compiler/Xcode/SDK, flags or target changes select another cache namespace. Historical producer identities remain bound to the imported object instead of the present contents of consumed archive paths. Identical inputs reuse Dune/Swift incremental state. Nothing is automatically deleted. The cache root has a nonblocking process lock; native builds also hold the opam switch read lock. Do not point at another task's legacy build/cache/signing directories; coordinate with its owner first. The script owns only its external cache and new output directory; it refuses an existing `--app-dir` instead of removing it. -- `--configuration debug|release`, `--deployment-target` (>=26.0), `--jobs`, `--version` and `--build-number` are explicit inputs. Unset ambient `SDKROOT`/compiler/OCaml lookup overrides; the Swift manifest runs on macOS and the selected target SDK is passed explicitly. Production native objects must retain mldoc and XMLm symbols: merely listing an unused `.cmxa` does not prove retention. A missing unit stops the build for linker-rule review; ask before changing protected Dune rules. -- Resources and C4 icons are packaged before signing. Device mode checks the existing iOS profile, expiration, certificate, team/bundle and allowed entitlements, then reads back the actual signature/entitlements. Simulator uses adhoc signing with entitlements in `__TEXT`; macOS development signing takes `--sign-identity` and `JOURNAL_MACOS_TEAM_ID`. Version flags affect only the generated plist. Build never installs or claims startup acceptance. -- Experimental mode checks every Swift object against the selected Mach-O platform, checks the final DNS/crypto link-map provider is its fresh target C object, and requires runtime/mldoc/XMLm symbols. Its report and generated plist explicitly mark host-restamped runtime origin and `cross_abi_validated=false`. This reproduces the existing limited route, not full OCaml iOS cross-compilation. Raw `JOURNAL_OCAML_OBJECT`/extra-object injection and the old object-injecting Dune aliases remain refused; use the explicit commands above. Any protected Dune/spec/lock edit still needs separate approval. +Use `bash tool/build_journal_apple.sh --help` as the Apple build entry point. Read [journal-build](.agents/skills/journal-build/SKILL.md) for target selection, prerequisites, cache/provenance, failure handling and acceptance limits. diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index ef9dd2e2..5fe50e16 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -32,6 +32,7 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route ## Consequences - The existing shell entry point delegates to one standard-library Python implementation. The documented entry point replaces legacy object-injecting aliases for supported builds; protected Dune/spec/lock files are unchanged. +- At the user's request, AGENTS retains only the build entry and skill link. Target/prerequisite, provenance/cache, failure and acceptance guidance lives once in `.agents/skills/journal-build/SKILL.md`, following skill-creator guidance and the repository's skill format. This documentation-only reorganization preserves the verified builder bytes and needs no repeat production build. - Twenty boundary tests pass on Python 3.9 and 3.13, including changed/missing consumed archives, a later builder-only commit and negative object/source/C/header/evidence/receipt/SDK cases. Two actual Simulator link-only assemblies passed target, resource/icon and strict signing checks, and reused one cache namespace. Concurrent entry was rejected as busy. Icon packaging passed for all three targets. - Independent exact-head review reproduced a cancellation race: a leader could exit on SIGTERM while its descendant ignored it, allowing the cache lock to release early. Cancellation now kills the remaining process group even after the leader exits. A real descendant holding a FIFO write endpoint verifies termination before lock release; it failed before the fix and passes afterward. - The final independent review also reproduced untracked Swift source being included by SwiftPM while the LUI pin guard ignored it. Production LUI checks now include all untracked files; a preflight regression rejects the added Swift source before native import. The successful production builds used a checkout without this trigger. From 3419259c24216f2061889e5151bb7885524d6cf9 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 11:54:54 +0800 Subject: [PATCH 5/7] Fix switch pins, preserve probe workflows and verify GMP inputs --- .agents/skills/journal-build/SKILL.md | 10 +- .../2026-10-15-reusable-apple-build.md | 1 + tool/journal_build.py | 92 ++++++++---- tool/lui_probe_host.py | 20 ++- tool/test_journal_build.py | 137 ++++++++++++++++++ tool/test_swiftui_warm_start.py | 5 +- 6 files changed, 225 insertions(+), 40 deletions(-) diff --git a/.agents/skills/journal-build/SKILL.md b/.agents/skills/journal-build/SKILL.md index fd2a629c..2be5f318 100644 --- a/.agents/skills/journal-build/SKILL.md +++ b/.agents/skills/journal-build/SKILL.md @@ -17,7 +17,9 @@ All targets are arm64 with deployment target >=26.0. Choose an external `--cache | `ios-simulator` | `iphonesimulator` | Explicit `--link-only` stub for Swift/link/package checks, or the experimental mode below. | | `ios-device` | `iphoneos` | Explicit `--experimental-host-restamp --native-manifest FILE`, existing certificate SHA1/profile/entitlements. | -Default iOS runtime builds and raw `JOURNAL_OCAML_OBJECT`/extra-object injection are refused. Host ARM64 OCaml/runtime restamping is an experimental compatibility route, **not verified full iOS cross-compilation or ABI acceptance**. Real target C/Swift and signing checks do not establish full OCaml ABI or startup acceptance. +Default iOS runtime builds and raw `JOURNAL_OCAML_OBJECT`/extra-object injection are refused. Existing `lui_probe_host.py` callers keep `--native-object`: the helper selects isolated `--probe-native-object` or explicit `--link-only` when absent, with its own host-local cache. Probe import requires external staged Swift/plist, a distinct `-probe` bundle ID, a matching macOS/Simulator object and runtime/bridge symbols; it hashes the object before/after building. It never restamps, accepts device builds or claims production source/ABI proof. Warm-start Simulator inputs must already match Simulator. + +Host ARM64 OCaml/runtime restamping is an experimental compatibility route, **not verified full iOS cross-compilation or ABI acceptance**. Real target C/Swift and signing checks do not establish full OCaml ABI or startup acceptance. Simulator signing is adhoc. macOS development signing needs `--sign-identity` plus `JOURNAL_MACOS_TEAM_ID`; device mode requires the existing signing inputs shown below. @@ -33,7 +35,7 @@ bash tool/build_journal_apple.sh ios-device --experimental-host-restamp \ --sign-entitlements /path/to/device.entitlements --build-number 1.2.3 ``` -`--preflight` checks without compilation; remove it for the macOS build. Unset ambient compiler/OCaml/SDK lookup overrides; SDK selection is explicit. Provision dependencies separately with the existing `tool/prepare_ci_dependencies.py` lock/resolved-pin workflow, which fetches and pins; use only an authorized switch. The builder never repairs it implicitly. Offline moving refs have no remote freshness claim. +`--preflight` checks without compilation; remove it for the macOS build. Unset ambient compiler/OCaml/SDK lookup overrides; SDK selection is explicit. Pins come from active switch state; raw source metadata may omit `url`, while any recorded installed source must still agree. CI local pins require their matching resolution record and clean resolved commit. Provision dependencies separately with `tool/prepare_ci_dependencies.py`, which fetches and pins; use only an authorized switch. The builder never repairs it implicitly. Offline moving refs have no remote freshness claim. ## Experimental object provenance @@ -59,10 +61,10 @@ The producer commit must be an ancestor of the builder revision. The full native Identity includes source, actual dependency ABI inputs, toolchain/SDK, flags and target. Identical inputs reuse incremental state; changed inputs select another namespace. Every run leaves external phase logs and `run-*/manifest.json`. The cache has a nonblocking lock; native Dune builds also hold the switch read lock. A busy lock belongs to its active owner: coordinate or wait. Preserve existing output/rollback apps and choose a new `--app-dir`; nothing is cleared automatically. -For failure, read `failed_phase`, `error` and that phase's log. Correct target/environment or select matching prepared inputs. Source/configuration/pin/current-input mismatch requires matching provenance; a historical archive change alone can use the validated receipt contract above. Missing mldoc/XMLm retention requires linker-rule review; protected Dune/spec/lock edits still follow the repository's authorization rules. Legacy object-injecting Dune aliases/probe callers need migration to this entry point. +For failure, read `failed_phase`, `error` and that phase's log. Correct target/environment or select matching prepared inputs. Source/configuration/pin/current-input mismatch requires matching provenance; a historical archive change alone can use the validated receipt contract above. Missing mldoc/XMLm retention requires linker-rule review; protected Dune/spec/lock edits still follow the repository's authorization rules. Legacy object-injecting Dune aliases remain refused; probe helpers use the isolated mode above. ## Accept and hand off -Only `status: built` confirms an artifact. Check recorded revisions/dirty hashes, target/SDK/flags, bundle/executable hashes, version and signing identity. Resources/icons precede signing; device profile/certificate/team/entitlements and strict signature are checked. Experimental builds additionally check Swift object platforms, fresh target C link-map providers and runtime/mldoc/XMLm retention. Listing a `.cmxa` alone does not establish retention. +Only `status: built` confirms an artifact. Check recorded revisions/dirty hashes, target/SDK/flags, bundle/executable hashes, version and signing identity. Resources/icons precede signing; device profile/certificate/team/entitlements and strict signature are checked. Current GMP archives are rehashed before native macOS acceptance. Experimental builds additionally check Swift object platforms, fresh target C link-map providers and runtime/mldoc/XMLm retention. Listing a `.cmxa` alone does not establish retention. `preflight-passed` and `link-only` are not runtime acceptance. Preserve `cross_abi_validated=false` for host-restamped objects and state which compilation was reused. Carry the manifest with the App; installation and phone/graph operations are separate from build. For builder changes, use `PYTHONDONTWRITEBYTECODE=1 python3 tool/test_journal_build.py` and the existing icon test as appropriate. diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index 5fe50e16..a414c6fa 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -39,6 +39,7 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route - An actual experimental iPhoneOS build produced a signed production App: runtime/mldoc/XMLm retained, 55 Swift objects verified as iOS ARM64, DNS/crypto link-map providers verified as freshly compiled iPhoneOS C, and profile/signature/entitlements read back successfully. Subsequent review tightened source deletion/configuration, compiler and profile guards. A final retry correctly stopped before compilation when another authorized task changed 14 historical producer archive inputs; it did not accept the old provenance or interfere with that cache. The successful App remains preserved. Independent journal-code-review found no remaining substantive issue after fixes. - The later archive mismatch exposed an overly strict importer lifecycle rule: the fixed complete object remained valid while another owner normally updated its consumed archives. Receipt-bound historical provenance now distinguishes that state from changed current inputs. After the repair, a final production iPhoneOS build passed through this entry point: fixed OCaml complete object reused, target C and all Swift/LUI objects compiled in a new cache namespace, icons/resources packaged, signature/entitlements and input stability verified. No OCaml recompilation or phone operation occurred. - Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. +- GitHub review exposed three further build-owner gaps: raw opam metadata was incorrectly used as the active pin, legacy probe helpers still injected a rejected environment object, and GMP lacked final rehashing. Active switch pins now drive lock/CI resolution (including local Git pins), optional installed source metadata remains a conflict guard, and the recorded GMP archive is rehashed before acceptance. Existing probe callers retain `--native-object` through a shared isolated probe mode with explicit target/runtime/hash checks and no production provenance/ABI claim; it cannot build device or production bundles. The four added boundary tests use a real offline temporary opam switch, tiny real Mach-O objects and the actual helper/CLI. These filesystem/CLI failures have no application reducer owner; no product or OCaml test is added. ## Questions diff --git a/tool/journal_build.py b/tool/journal_build.py index 1a731dfb..0d25800a 100644 --- a/tool/journal_build.py +++ b/tool/journal_build.py @@ -17,6 +17,7 @@ import subprocess import sys import tempfile +from urllib.parse import unquote, urlsplit ROOT = Path(__file__).resolve().parents[1] @@ -105,7 +106,8 @@ def __init__(self, args, run_dir): # They must not affect the explicit target SDK or switch in subprocesses. ignored = {name: self.env.pop(name) for name in ("SDKROOT", "CPATH", "LIBRARY_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH") if name in self.env} - mode = "experimental-host-restamp" if args.experimental_host_restamp else "link-only" if args.link_only else "native" + mode = ("probe-object" if getattr(args, "probe_native_object", None) else + "experimental-host-restamp" if args.experimental_host_restamp else "link-only" if args.link_only else "native") self.report = {"status": "running", "started": datetime.now(timezone.utc).isoformat(), "commands": [], "mode": mode, "cross_abi_validated": False, "runtime_acceptance": False, "installed": False, "ignored_environment": ignored} @@ -183,17 +185,22 @@ def dependencies(self): if installed.get(name) != version: continue metadata = self.prefix / ".opam-switch/packages" / package / "opam" - actual = self.opam("show", "--just-file", "--field=url.src", metadata).strip('"') - if current_pins.get(name) != actual: - errors.append(f"{name}: active pin {current_pins.get(name)!r} differs from installed source {actual!r}") + actual = current_pins.get(name, "") + installed_source = self.opam("show", "--just-file", "--field=url.src", metadata).strip('"') + # Raw source metadata may omit url; the pin belongs to switch state. + # When recorded, installed provenance must still agree with that state. + if installed_source and installed_source != actual: + errors.append(f"{name}: active pin {actual!r} differs from installed source {installed_source!r}") checkout = self.prefix / ".opam-switch/sources" / name ref = expected.rsplit("#", 1)[1] if actual != expected: record = ci.get(package, {}) checkout = (ROOT / record.get("checkout", "missing-ci-checkout")).resolve() # A local pin needs the existing CI resolution record, including exact source URL. - if record.get("source") != expected or actual.removeprefix("file://") != str(checkout): - errors.append(f"{name}: installed pin {actual!r} differs from lock {expected!r}") + target = urlsplit(actual.removeprefix("git+")) + local_path = unquote(target.path) if target.scheme == "file" and not target.netloc else actual + if record.get("source") != expected or local_path != str(checkout): + errors.append(f"{name}: active pin {actual!r} differs from lock {expected!r}") continue ref = record.get("resolved_sha", "") if not (checkout / ".git").exists(): @@ -207,7 +214,8 @@ def dependencies(self): continue if dirty or (re.fullmatch(r"[0-9a-f]{40}", ref) and head != ref): errors.append(f"{name}: dirty or wrong resolved source: {head}; expected {ref}") - resolved[package] = {"source": expected, "resolved_sha": head, "checkout": str(checkout)} + resolved[package] = {"source": expected, "active_pin": actual, + "resolved_sha": head, "checkout": str(checkout)} self.report["dependency_errors"] = errors self.report["dependencies"] = {"installed": installed, "pins": resolved, "freshness": "offline: moving refs use the currently installed resolved checkout; no remote update"} @@ -371,6 +379,7 @@ def producer_files(self, pre, link, xmlm): def preflight(self): a = self.args + probe = getattr(a, "probe_native_object", None) if self.run("uname", "-m") != "arm64": raise RuntimeError("only Darwin arm64 is supported") sdk_name = {"macos": "macosx", "ios-simulator": "iphonesimulator", "ios-device": "iphoneos"}[a.platform] @@ -390,7 +399,7 @@ def preflight(self): raise RuntimeError(f"LUI Apple package missing: {self.lui}; set --lui-package") # Production Swift LUI must agree with the OCaml lock; link-only may intentionally inspect another backend. lui_sha = self.lui_sha = self.run("git", "-C", self.lui, "rev-parse", "HEAD") - if not a.link_only: + if not (a.link_only or probe): expected = re.search(r'"git\+https://github.com/logseq/lui.git#([0-9a-f]{40})"', (ROOT / "logseq_journal.opam.locked").read_text()).group(1) if lui_sha != expected or self.run("git", "-C", self.lui, "status", "--porcelain", "--untracked-files=all"): @@ -401,6 +410,9 @@ def preflight(self): raise RuntimeError(f"missing {key}: {self.plist}") if self.info["CFBundleExecutable"] != "JournalApp": raise RuntimeError("CFBundleExecutable must be JournalApp") + if probe and (not self.info["CFBundleIdentifier"].endswith("-probe") + or any(path == ROOT or ROOT in path.parents for path in (self.swift_dir, self.plist))): + raise RuntimeError("probe mode requires external staged Swift/plist and a distinct -probe bundle ID") if a.version: self.info["CFBundleShortVersionString"] = a.version if a.build_number: @@ -420,7 +432,17 @@ def preflight(self): self.tools = tools source = source_inputs() dependencies = {"mode": "stub; OCaml dependencies not used or validated"} - if a.native_manifest: + if probe: + probe = probe.resolve() + self.verify_target(probe) + symbols = self.run("nm", "-U", probe) + if any(not re.search(r"\b_" + name + r"$", symbols, re.M) + for name in ("caml_startup", "lui_ocaml_start")): + raise RuntimeError("probe object must define OCaml runtime and LUI bridge entry points") + self.native_files = {str(probe): sha(probe)} + dependencies = {"mode": "caller-supplied probe object; no production source or OCaml ABI proof", + "files": self.native_files, "production_source_validated": False} + elif a.native_manifest: dependencies = self.imported_native(source) elif not a.link_only: self.prefix = Path(self.opam("var", "prefix")) @@ -452,6 +474,8 @@ def compile(self, workspace): native = workspace / "journal_stub.o" self.run(self.tools["clang"], "-target", self.triple, "-isysroot", self.sdk, "-fPIC", "-c", source, "-o", native) + elif a.probe_native_object: + native = a.probe_native_object.resolve() elif a.native_manifest: native = workspace / "journal_complete_restamped.o" platform = "2" if a.platform == "ios-device" else "7" @@ -586,6 +610,9 @@ def assemble(self, product, app): JournalBuildInputDigest=self.report["cache_key"], JournalBuildMode=self.report["mode"]) if self.args.experimental_host_restamp: info.update(JournalOCamlRuntimeOrigin="host-arm64-restamp", JournalCrossABIValidated=False) + if self.report["mode"] == "probe-object": + info.update(JournalOCamlRuntimeOrigin="caller-supplied-probe", JournalCrossABIValidated=False, + JournalProductionSourceValidated=False) info["LSMinimumSystemVersion" if mac else "MinimumOSVersion"] = self.args.deployment_target info.setdefault("CFBundleURLTypes", []).append({"CFBundleURLName": "com.logseq.journal.auth", "CFBundleURLSchemes": ["logseqjournal"]}) @@ -635,6 +662,27 @@ def assemble(self, product, app): self.report["artifact"] = {"app": str(app), "info": packaged, "files": files(app), "signature": signature, "executable_sha256": sha(executable), "sign_identity": self.args.sign_identity} + def verify_inputs(self): + self.phase = "verify-inputs" + if (source_inputs() != self.inputs["source"] or files(self.swift_dir) != self.inputs["swift"] + or files(self.lui) != self.inputs["lui"] or sha(self.plist) != self.inputs["plist"] + or files(self.entitlements) != self.inputs["entitlements"]): + raise RuntimeError("source changed during build; artifact is not accepted, rerun with stable inputs") + if any(sha(path) != self.inputs["toolchain"]["hashes"][name] for name, path in self.tools.items()): + raise RuntimeError("Apple toolchain changed during build; artifact is not accepted") + if any(sha(p) != h for p, h in self.inputs["toolchain"]["sdk_identity"]["settings"].items()): + raise RuntimeError("SDK identity changed during build; artifact is not accepted") + gmp = self.inputs["toolchain"].get("gmp") + if gmp and sha(gmp["path"]) != gmp["sha256"]: + raise RuntimeError("GMP archive changed during build; artifact is not accepted") + imported = getattr(self, "native_files", {}) + if any(sha(p) != h for p, h in {**imported, **self.inputs["signing"]}.items()): + raise RuntimeError("imported native/signing inputs changed during build; artifact is not accepted") + a = self.args + if not (a.link_only or a.native_manifest or getattr(a, "probe_native_object", None)): + if self.dependencies() != self.inputs["dependencies"]: + raise RuntimeError("switch changed during build; artifact is not accepted") + def arguments(): parser = argparse.ArgumentParser(description="Build Journal Apple host; never installs or updates dependencies.") @@ -647,6 +695,7 @@ def arguments(): parser.add_argument("--link-only", action="store_true", help="explicit stub build; not runnable acceptance") parser.add_argument("--experimental-host-restamp", action="store_true", help="explicit existing host ARM64 compatibility route; full iOS ABI unverified") parser.add_argument("--native-manifest", type=Path, help="hash-verified source/link/XMLm producer evidence envelope") + parser.add_argument("--probe-native-object", type=Path, help="isolated macOS/Simulator probe only; target-checked object, no production source/ABI proof") parser.add_argument("--cache-dir", type=Path, default=Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "logseq-journal-build") parser.add_argument("--configuration", choices=("debug", "release"), default="debug") parser.add_argument("--jobs", type=int, default=2) @@ -657,19 +706,21 @@ def arguments(): parser.add_argument("--provisioning-profile", type=Path, help="existing device profile; never downloaded") parser.add_argument("--sign-entitlements", type=Path, help="existing device signing entitlements") a = parser.parse_args() + if a.probe_native_object and (a.platform == "ios-device" or a.switch or a.link_only or a.native_manifest or a.experimental_host_restamp): + parser.error("probe mode is isolated macOS/Simulator object import, separate from production/stub modes") for name in ("JOURNAL_OCAML_OBJECT", "JOURNAL_EXTRA_OBJECTS", "JOURNAL_NATIVE_LINK_INPUTS"): if os.environ.get(name): parser.error(f"{name} is unproven input; use native macOS --switch or explicit --link-only. " - "Legacy Dune aliases/probe object injection need provenance support; host restamp is not cross compilation.") + "Probe helpers use isolated --probe-native-object; host restamp is not cross compilation.") if bool(a.native_manifest) != a.experimental_host_restamp: parser.error("--native-manifest and --experimental-host-restamp must be supplied together") if a.experimental_host_restamp and (a.link_only or a.switch or a.platform == "macos"): parser.error("experimental import is an iOS mode, separate from --link-only and --switch") if a.platform == "ios-device" and not a.experimental_host_restamp: parser.error("iPhone build blocked: no verified complete OCaml iOS cross toolchain/ABI route; host restamp is unsupported") - if a.platform == "ios-simulator" and not (a.link_only or a.experimental_host_restamp): + if a.platform == "ios-simulator" and not (a.link_only or a.experimental_host_restamp or a.probe_native_object): parser.error("Simulator runtime build blocked; use --link-only for Swift/link checks. Host restamp is not verified cross compilation.") - if not (a.link_only or a.native_manifest or a.switch): + if not (a.link_only or a.native_manifest or a.probe_native_object or a.switch): parser.error("native macOS requires --switch; no implicit switch/pin changes") if a.platform == "ios-simulator" and a.sign_identity != "-": parser.error("Simulator requires adhoc signature; entitlements live in __TEXT") @@ -705,7 +756,7 @@ def interrupted(signum, frame): run_dir = Path(tempfile.mkdtemp(prefix="run-", dir=a.cache_dir)) build = Build(a, run_dir) try: - if not a.link_only and not a.native_manifest: + if not (a.link_only or a.native_manifest or a.probe_native_object): prefix = Path(build.opam("var", "prefix")) # opam uses POSIX record locks. Keep its switch read-locked for the whole build. switch_lock = stack.enter_context((prefix / ".opam-switch/lock").open("rb")) @@ -721,20 +772,7 @@ def interrupted(signum, frame): (workspace / "inputs.json").write_text(json.dumps(build.inputs, indent=2) + "\n") product = build.compile(workspace) build.assemble(product, a.app_dir or run_dir / "LogseqJournal.app") - build.phase = "verify-inputs" - if (source_inputs() != build.inputs["source"] or files(build.swift_dir) != build.inputs["swift"] - or files(build.lui) != build.inputs["lui"] or sha(build.plist) != build.inputs["plist"] - or files(build.entitlements) != build.inputs["entitlements"]): - raise RuntimeError("source changed during build; artifact is not accepted, rerun with stable inputs") - if any(sha(path) != build.inputs["toolchain"]["hashes"][name] for name, path in build.tools.items()): - raise RuntimeError("Apple toolchain changed during build; artifact is not accepted") - if any(sha(p) != h for p, h in build.inputs["toolchain"]["sdk_identity"]["settings"].items()): - raise RuntimeError("SDK identity changed during build; artifact is not accepted") - imported = getattr(build, "native_files", {}) - if any(sha(p) != h for p, h in {**imported, **build.inputs["signing"]}.items()): - raise RuntimeError("imported native/signing inputs changed during build; artifact is not accepted") - if not a.link_only and not a.native_manifest and build.dependencies() != build.inputs["dependencies"]: - raise RuntimeError("switch changed during build; artifact is not accepted") + build.verify_inputs() build.report["status"] = "preflight-passed" if a.preflight else "built" build.report["finished"] = datetime.now(timezone.utc).isoformat() build.save() diff --git a/tool/lui_probe_host.py b/tool/lui_probe_host.py index a1fde2de..10b28344 100644 --- a/tool/lui_probe_host.py +++ b/tool/lui_probe_host.py @@ -10,8 +10,11 @@ create the complete object with `ocamlfind ocamlopt -linkpkg -output-complete-obj` over the `app` archive plus the probe module (the probe self-registers via `Journal_bridge.register`, so link it last so its -registration wins). Pass the result via `--native-object` / JOURNAL_OCAML_OBJECT; -without it the build links the stub object (link-only validation). +registration wins). Pass the result via `--native-object`; the helper selects +isolated `--probe-native-object`, records its hash and checks its target/runtime +entries. Supply an object already built for macOS or Simulator; no restamping +is done. This records caller-supplied probe input, not production source or ABI +proof. Without it the build explicitly selects stub link-only validation. """ import os from pathlib import Path @@ -46,10 +49,10 @@ def stage(host, app_swift, bundle_id, display_name, swift_names=None): def build(host, platform="macos", app_name="JournalApp.app", native_object=None, - extra_env=None): + extra_env=None, preflight=False): """Build the staged host through tool/build_journal_apple.sh. Returns the assembled .app path. `platform` is 'macos' or 'ios-simulator'.""" - host = Path(host) + host = Path(host).resolve() env = dict( os.environ, JOURNAL_SWIFT_DIR=str(host / "swift"), @@ -59,11 +62,14 @@ def build(host, platform="macos", app_name="JournalApp.app", native_object=None, str((ROOT / "../lui/platform/apple").resolve())), ) env.update(extra_env or {}) - if native_object: - env["JOURNAL_OCAML_OBJECT"] = str(Path(native_object).resolve()) + mode = (["--probe-native-object", str(Path(native_object).resolve())] + if native_object else ["--link-only"]) + if preflight: + mode.append("--preflight") app_dir = host / "apple" / app_name result = subprocess.run( - ["bash", str(ROOT / "tool" / "build_journal_apple.sh"), platform, str(app_dir)], + ["bash", str(ROOT / "tool" / "build_journal_apple.sh"), platform, str(app_dir), + "--cache-dir", str(host / "build-cache"), *mode], cwd=ROOT, env=env, capture_output=True, text=True) print(result.stdout, end="") print(result.stderr, end="") diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py index 7fdbc4b6..bd70ddad 100644 --- a/tool/test_journal_build.py +++ b/tool/test_journal_build.py @@ -65,6 +65,15 @@ def test_native_import_requires_both_explicit_mode_and_manifest(self): self.assertNotEqual(result.returncode, 0) self.assertIn(expected, result.stderr) + def test_probe_object_cannot_select_device_or_production_modes(self): + for args in [('ios-device', '--probe-native-object', '/tmp/probe.o'), + ('macos', '--probe-native-object', '/tmp/probe.o', '--link-only'), + ('macos', '--probe-native-object', '/tmp/probe.o', '--switch', 'unused')]: + with self.subTest(args=args): + result = self.cli(*args, '--preflight') + self.assertNotEqual(result.returncode, 0) + self.assertIn('probe mode', result.stderr) + def test_foreign_dune_root_and_runtime_cannot_change_provenance(self): for name in ("DUNE_ROOT", "DUNE_WORKSPACE", "DUNE_CROSS_TARGET", "DUNE_INSTRUMENT_WITH", "OCAMLLIB", "MACOSX_DEPLOYMENT_TARGET", "CAML_LD_LIBRARY_PATH", "OCAMLFIND_COMMANDS"): @@ -96,6 +105,134 @@ def setUpClass(cls): cls.build = importlib.util.module_from_spec(spec) spec.loader.exec_module(cls.build) + @contextmanager + def pinned_switch(self): + """Real offline switch; original source metadata deliberately has no url.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + repository, checkout, journal = (root / name for name in ('repository', 'checkout', 'journal')) + for path in (repository, checkout, journal): + path.mkdir() + (repository / 'repo').write_text('opam-version: "2.0"\n') + metadata = 'opam-version: "2.0"\nversion: "dev"\nsynopsis: "Offline build fixture"\n' + (checkout / 'fixture.opam').write_text(metadata) + for argv in (['git', 'init', '-q', checkout], ['git', '-C', checkout, 'add', '.'], + ['git', '-C', checkout, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'fixture']): + subprocess.run(argv, check=True, capture_output=True) + commit = subprocess.check_output(['git', '-C', checkout, 'rev-parse', 'HEAD'], text=True).strip() + expected = 'git+https://example.invalid/fixture.git#' + commit + (journal / 'journal.opam').write_text('opam-version: "2.0"\n') + lock = journal / 'journal.opam.locked' + lock.write_text('depends: [ "fixture" {= "dev"} ]\npin-depends: [ [ "fixture.dev" "' + expected + '" ] ]\n') + (journal / '.ci-reports').mkdir() + (journal / '.ci-reports/dependencies.json').write_text(json.dumps([ + dict(package='fixture.dev', source=expected, checkout=str(checkout), resolved_sha=commit)])) + env = dict(os.environ, OPAMROOT=str(root / 'opam'), OPAMYES='1', OPAMNOENVNOTICE='1') + for argv in (['opam', 'init', '--bare', '--no-setup', '--no-opamrc', '--disable-sandboxing', 'local', str(repository)], + ['opam', 'switch', 'create', 'fixture', '--empty'], + ['opam', 'pin', 'add', '--switch', 'fixture', '--no-action', 'fixture.dev', str(checkout)], + ['opam', 'install', '--switch', 'fixture', 'fixture.dev', '--assume-depexts']): + subprocess.run(argv, env=env, check=True, capture_output=True) + build = self.build.Build(SimpleNamespace(link_only=False, experimental_host_restamp=False, + native_manifest=None, switch='fixture'), root) + build.env = env + build.prefix = root / 'opam/fixture' + # Keep the active pin intact while storing the original no-url opam file. + (build.prefix / '.opam-switch/packages/fixture.dev/opam').write_text(metadata) + with patch.object(self.build, 'ROOT', journal): + yield SimpleNamespace(root=root, journal=journal, checkout=checkout, lock=lock, build=build) + + def test_active_switch_pin_does_not_require_raw_metadata_url(self): + with self.pinned_switch() as f: + dependencies = f.build.dependencies() + self.assertEqual(dependencies['pins']['fixture.dev']['checkout'], str(f.checkout)) + metadata = f.build.prefix / '.opam-switch/packages/fixture.dev/opam' + original = metadata.read_text() + metadata.write_text(original + 'url { src: "git+https://example.invalid/stale" }\n') + with self.assertRaisesRegex(RuntimeError, 'installed source'): + f.build.dependencies() + metadata.write_text(original) + f.lock.write_text(f.lock.read_text().replace('example.invalid/fixture', 'example.invalid/other')) + with self.assertRaisesRegex(RuntimeError, 'differs from lock'): + f.build.dependencies() + + def test_final_verification_rejects_changed_gmp_and_probe_input(self): + self.assertTrue(hasattr(self.build.Build, 'verify_inputs'), 'final input verification boundary missing') + with self.pinned_switch() as f: + build = f.build + build.swift_dir, build.lui, build.entitlements = (f.root / name for name in ('swift', 'lui', 'entitlements')) + for path in (build.swift_dir, build.lui, build.entitlements): + path.mkdir() + build.plist = f.root / 'Info.plist' + build.plist.write_bytes(b'plist fixture') + gmp, probe = f.root / 'libgmp.a', f.root / 'probe.o' + gmp.write_bytes(b'original GMP bytes') + probe.write_bytes(b'original probe bytes') + build.tools = {} + build.native_files = {str(probe): self.build.sha(probe)} + build.inputs = dict(source=self.build.source_inputs(), swift={}, lui={}, entitlements={}, + plist=self.build.sha(build.plist), signing={}, dependencies=build.dependencies(), + toolchain=dict(hashes={}, sdk_identity={'settings': {}}, + gmp=dict(path=str(gmp), sha256=self.build.sha(gmp)))) + build.verify_inputs() + gmp.write_bytes(b'changed GMP bytes') + with self.assertRaisesRegex(RuntimeError, 'GMP'): + build.verify_inputs() + gmp.write_bytes(b'original GMP bytes') + probe.write_bytes(b'changed probe bytes') + with self.assertRaisesRegex(RuntimeError, 'native'): + build.verify_inputs() + + def test_probe_helper_preflight_supports_stub_and_platform_matched_objects(self): + sys.path.insert(0, str(ROOT / 'tool')) + import lui_probe_host + import inspect + self.assertIn('preflight', inspect.signature(lui_probe_host.build).parameters, + 'probe helper has no reusable preflight path') + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + lui = root / 'lui' + lui.mkdir() + (lui / 'Package.swift').write_text('// preflight only\n') + for argv in (['git', 'init', '-q', lui], ['git', '-C', lui, 'add', '.'], + ['git', '-C', lui, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'fixture']): + subprocess.run(argv, check=True, capture_output=True) + source = root / 'probe.c' + source.write_text('void caml_startup(void) {}\nvoid lui_ocaml_start(void) {}\n') + for platform, triple in [('macos', 'arm64-apple-macosx26.0'), + ('ios-simulator', 'arm64-apple-ios26.0-simulator')]: + obj = root / (platform + '.o') + subprocess.run(['xcrun', 'clang', '-target', triple, '-c', source, '-o', obj], + check=True, capture_output=True) + for native in (None, obj): + host = root / (platform + ('-native' if native else '-stub')) + lui_probe_host.stage(host, '/* preflight only */', 'fixture.journal-probe', 'Probe') + lui_probe_host.build(host, platform=platform, native_object=native, preflight=True, + extra_env={'JOURNAL_LUI_PACKAGE_PATH': str(lui)}) + manifest = json.loads(next((host / 'build-cache').glob('run-*/manifest.json')).read_text()) + self.assertEqual(manifest['status'], 'preflight-passed') + self.assertEqual(manifest['mode'], 'probe-object' if native else 'link-only') + if platform == 'ios-simulator': + with self.assertRaises(SystemExit): + lui_probe_host.build(root / (platform + '-native'), platform=platform, + native_object=root / 'macos.o', preflight=True, + extra_env={'JOURNAL_LUI_PACKAGE_PATH': str(lui)}) + host = root / 'production-id' + lui_probe_host.stage(host, '/* preflight only */', 'org.logseq.journal', 'Production') + with self.assertRaises(SystemExit): + lui_probe_host.build(host, native_object=root / 'macos.o', preflight=True, + extra_env={'JOURNAL_LUI_PACKAGE_PATH': str(lui)}) + source.write_text('int unrelated_object;\n') + subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-macosx26.0', '-c', source, + '-o', root / 'unrelated.o'], check=True, capture_output=True) + host = root / 'missing-runtime' + lui_probe_host.stage(host, '/* preflight only */', 'fixture.journal-probe', 'Probe') + with self.assertRaises(SystemExit): + lui_probe_host.build(host, native_object=root / 'unrelated.o', preflight=True, + extra_env={'JOURNAL_LUI_PACKAGE_PATH': str(lui)}) + def test_source_toolchain_dependency_flags_and_target_separate_caches(self): self.assertTrue(hasattr(self, "build"), "build input fingerprint missing") baseline = dict(source="a", dependencies="b", toolchain="c", flags=[], target="macos") diff --git a/tool/test_swiftui_warm_start.py b/tool/test_swiftui_warm_start.py index 9266ab0c..5854a936 100644 --- a/tool/test_swiftui_warm_start.py +++ b/tool/test_swiftui_warm_start.py @@ -28,7 +28,7 @@ parser.add_argument('--history-days', type=int, default=0, help='Additional journal days with 12 rows each (0–30)') parser.add_argument('--references', action='store_true', help='Seed isolated UUID reference and literal fallback cases') parser.add_argument('--platform', choices=['macos', 'ios-simulator'], default='macos') -parser.add_argument('--native-object', type=Path, help='Use the current verified production object') +parser.add_argument('--native-object', type=Path, help='Current complete object already matching the selected macOS/Simulator target') arguments = parser.parse_args() if not 1 <= arguments.rows <= 5000: parser.error('--rows must be between 1 and 5000') @@ -60,7 +60,8 @@ def run(command, cwd=host): # The OCaml side ships the production complete object: app/native_embed.exe.o. # Build it first with the workspace dune build (see tool/build_journal_apple.sh); -# pass a prebuilt artifact via --native-object. +# pass a prebuilt artifact via --native-object. Simulator needs a Simulator +# object; the shared builder does not silently restamp host runtime objects. artifact = arguments.native_object.resolve() if arguments.native_object else ( root/'_build/default/app/native_embed.exe.o') if not artifact.is_file(): From 7a85f7d2e48a26055be8aa3cbb9e0777b56d1261 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 11:59:33 +0800 Subject: [PATCH 6/7] Bind local pin refs and CI resolution to locked commits --- .../2026-10-15-reusable-apple-build.md | 1 + tool/journal_build.py | 15 ++++++++++- tool/test_journal_build.py | 25 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index a414c6fa..1f675ebf 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -40,6 +40,7 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route - The later archive mismatch exposed an overly strict importer lifecycle rule: the fixed complete object remained valid while another owner normally updated its consumed archives. Receipt-bound historical provenance now distinguishes that state from changed current inputs. After the repair, a final production iPhoneOS build passed through this entry point: fixed OCaml complete object reused, target C and all Swift/LUI objects compiled in a new cache namespace, icons/resources packaged, signature/entitlements and input stability verified. No OCaml recompilation or phone operation occurred. - Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. - GitHub review exposed three further build-owner gaps: raw opam metadata was incorrectly used as the active pin, legacy probe helpers still injected a rejected environment object, and GMP lacked final rehashing. Active switch pins now drive lock/CI resolution (including local Git pins), optional installed source metadata remains a conflict guard, and the recorded GMP archive is rehashed before acceptance. Existing probe callers retain `--native-object` through a shared isolated probe mode with explicit target/runtime/hash checks and no production provenance/ABI claim; it cannot build device or production bundles. The four added boundary tests use a real offline temporary opam switch, tiny real Mach-O objects and the actual helper/CLI. These filesystem/CLI failures have no application reducer owner; no product or OCaml test is added. +- Independent review of that pin repair found local URL refs and fixed lock commits needed separate checks. A valid CI SHA must match a fixed locked SHA, a local URL fragment must resolve to the same commit, and the clean checkout must match that identity. Three real offline negative cases were RED before the additional checks; moving refs keep the stated offline resolution limitation. ## Questions diff --git a/tool/journal_build.py b/tool/journal_build.py index 0d25800a..84100acd 100644 --- a/tool/journal_build.py +++ b/tool/journal_build.py @@ -202,7 +202,20 @@ def dependencies(self): if record.get("source") != expected or local_path != str(checkout): errors.append(f"{name}: active pin {actual!r} differs from lock {expected!r}") continue - ref = record.get("resolved_sha", "") + resolved_sha = record.get("resolved_sha", "") + if not re.fullmatch(r"[0-9a-f]{40}", resolved_sha): + errors.append(f"{name}: invalid CI resolved_sha") + continue + if re.fullmatch(r"[0-9a-f]{40}", ref) and resolved_sha != ref: + errors.append(f"{name}: CI resolution differs from locked commit {ref}") + continue + if target.fragment: + selected = self.run("git", "-C", checkout, "rev-parse", "--verify", "--end-of-options", + unquote(target.fragment) + "^{commit}") + if selected != resolved_sha: + errors.append(f"{name}: active local pin selects {selected}, CI resolved {resolved_sha}") + continue + ref = resolved_sha if not (checkout / ".git").exists(): errors.append(f"{name}: resolved source checkout missing: {checkout}") continue diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py index bd70ddad..9a093fcb 100644 --- a/tool/test_journal_build.py +++ b/tool/test_journal_build.py @@ -153,6 +153,31 @@ def test_active_switch_pin_does_not_require_raw_metadata_url(self): with self.assertRaisesRegex(RuntimeError, 'installed source'): f.build.dependencies() metadata.write_text(original) + record_path = f.journal / '.ci-reports/dependencies.json' + records = json.loads(record_path.read_text()) + commit = records[0]['resolved_sha'] + records[0]['resolved_sha'] = '' + record_path.write_text(json.dumps(records)) + with self.subTest(case='invalid resolution'), self.assertRaisesRegex(RuntimeError, 'resolved'): + f.build.dependencies() + records[0]['resolved_sha'] = commit + record_path.write_text(json.dumps(records)) + (f.checkout / 'other').write_text('another real commit\n') + subprocess.run(['git', '-C', f.checkout, 'add', '.'], check=True, capture_output=True) + subprocess.run(['git', '-C', f.checkout, '-c', 'user.name=Build test', '-c', 'user.email=build@example.invalid', + 'commit', '-qm', 'other'], check=True, capture_output=True) + other = subprocess.check_output(['git', '-C', f.checkout, 'rev-parse', 'HEAD'], text=True).strip() + records[0]['resolved_sha'] = other + record_path.write_text(json.dumps(records)) + with self.subTest(case='fixed lock'), self.assertRaisesRegex(RuntimeError, 'locked commit'): + f.build.dependencies() + subprocess.run(['git', '-C', f.checkout, 'reset', '--hard', commit], check=True, capture_output=True) + records[0]['resolved_sha'] = commit + record_path.write_text(json.dumps(records)) + f.build.opam('pin', 'add', '--no-action', 'fixture.dev', 'git+file://' + str(f.checkout) + '#' + other) + metadata.write_text(original) + with self.subTest(case='active pin ref'), self.assertRaisesRegex(RuntimeError, 'active local pin'): + f.build.dependencies() f.lock.write_text(f.lock.read_text().replace('example.invalid/fixture', 'example.invalid/other')) with self.assertRaisesRegex(RuntimeError, 'differs from lock'): f.build.dependencies() From 293c05cc5a43b63e5bfbadd57a2414639cadfcb1 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 15 Oct 2026 12:09:58 +0800 Subject: [PATCH 7/7] Verify linked GMP copy against the declared input --- .../implemented/process/2026-10-15-reusable-apple-build.md | 1 + tool/journal_build.py | 5 ++++- tool/test_journal_build.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md index 1f675ebf..8471bb61 100644 --- a/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md +++ b/docs/agent-guide/implemented/process/2026-10-15-reusable-apple-build.md @@ -41,6 +41,7 @@ Do not copy historical device deployment scripts: their host ARM64 restamp route - Full native macOS validation is blocked by the current switch's missing mldoc, nonlocked XMLm and incomplete cached Git source objects. No dependency repair, full OCaml iOS ABI acceptance, phone operation or runtime acceptance is claimed. - GitHub review exposed three further build-owner gaps: raw opam metadata was incorrectly used as the active pin, legacy probe helpers still injected a rejected environment object, and GMP lacked final rehashing. Active switch pins now drive lock/CI resolution (including local Git pins), optional installed source metadata remains a conflict guard, and the recorded GMP archive is rehashed before acceptance. Existing probe callers retain `--native-object` through a shared isolated probe mode with explicit target/runtime/hash checks and no production provenance/ABI claim; it cannot build device or production bundles. The four added boundary tests use a real offline temporary opam switch, tiny real Mach-O objects and the actual helper/CLI. These filesystem/CLI failures have no application reducer owner; no product or OCaml test is added. - Independent review of that pin repair found local URL refs and fixed lock commits needed separate checks. A valid CI SHA must match a fixed locked SHA, a local URL fragment must resolve to the same commit, and the clean checkout must match that identity. Three real offline negative cases were RED before the additional checks; moving refs keep the stated offline resolution limitation. +- GMP acceptance also checks Dune's actual linked archive copy against the preflight digest, covering a pkg-config selection change while the original archive stays unchanged. The existing GMP boundary regression includes that RED→GREEN case; no additional test layer or protected Dune change is needed. ## Questions diff --git a/tool/journal_build.py b/tool/journal_build.py index 84100acd..23cbe917 100644 --- a/tool/journal_build.py +++ b/tool/journal_build.py @@ -509,6 +509,8 @@ def compile(self, workspace): "--profile", "dev" if a.configuration == "debug" else "release", "-j", str(a.jobs), "app/native_embed.exe.o") native = dune_dir / "default/app/native_embed.exe.o" + self.native_gmp = dune_dir / "default/app/libgmp.a" + self.report["linked_gmp"] = {"path": str(self.native_gmp), "sha256": sha(self.native_gmp)} symbols = self.run("nm", "-U", native) for library in ("Mldoc", "Xmlm"): if not re.search(r"caml" + library + r"(?:__|_)", symbols): @@ -686,7 +688,8 @@ def verify_inputs(self): if any(sha(p) != h for p, h in self.inputs["toolchain"]["sdk_identity"]["settings"].items()): raise RuntimeError("SDK identity changed during build; artifact is not accepted") gmp = self.inputs["toolchain"].get("gmp") - if gmp and sha(gmp["path"]) != gmp["sha256"]: + if gmp and (sha(gmp["path"]) != gmp["sha256"] + or sha(getattr(self, "native_gmp", gmp["path"])) != gmp["sha256"]): raise RuntimeError("GMP archive changed during build; artifact is not accepted") imported = getattr(self, "native_files", {}) if any(sha(p) != h for p, h in {**imported, **self.inputs["signing"]}.items()): diff --git a/tool/test_journal_build.py b/tool/test_journal_build.py index 9a093fcb..776b557e 100644 --- a/tool/test_journal_build.py +++ b/tool/test_journal_build.py @@ -193,6 +193,8 @@ def test_final_verification_rejects_changed_gmp_and_probe_input(self): build.plist.write_bytes(b'plist fixture') gmp, probe = f.root / 'libgmp.a', f.root / 'probe.o' gmp.write_bytes(b'original GMP bytes') + build.native_gmp = f.root / 'linked-libgmp.a' + build.native_gmp.write_bytes(gmp.read_bytes()) probe.write_bytes(b'original probe bytes') build.tools = {} build.native_files = {str(probe): self.build.sha(probe)} @@ -205,6 +207,10 @@ def test_final_verification_rejects_changed_gmp_and_probe_input(self): with self.assertRaisesRegex(RuntimeError, 'GMP'): build.verify_inputs() gmp.write_bytes(b'original GMP bytes') + build.native_gmp.write_bytes(b'different pkg-config-selected GMP copy') + with self.assertRaisesRegex(RuntimeError, 'GMP'): + build.verify_inputs() + build.native_gmp.write_bytes(gmp.read_bytes()) probe.write_bytes(b'changed probe bytes') with self.assertRaisesRegex(RuntimeError, 'native'): build.verify_inputs()