diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..95afd4e --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec pixi run pre-commit run --hook-stage pre-commit "$@" diff --git a/.github/workflows/con-pages-preview.yml b/.github/workflows/con-pages-preview.yml new file mode 100644 index 0000000..34f5ddf --- /dev/null +++ b/.github/workflows/con-pages-preview.yml @@ -0,0 +1,104 @@ +name: CON Pages preview + +on: + pull_request: + branches: + - main + push: + branches: + - main + - codex/milestone-3 + workflow_dispatch: + inputs: + deploy: + description: Deploy this exact ref to the github-pages environment + required: true + default: false + type: boolean + +permissions: + contents: read + +jobs: + build: + name: Build the backend-free CON artifact + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + concurrency: + group: con-pages-build-${{ github.ref }} + cancel-in-progress: true + + steps: + - name: Check out the pinned recursive source tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + submodules: recursive + + - name: Verify the accepted clean-migration checkpoints + shell: bash + run: | + set -euo pipefail + readonly parent_checkpoint=f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d + readonly site_checkpoint=a122e506de9e4a13473edbe8d74a950d74032a16 + readonly checkpoint_ref=refs/remotes/origin/codex/clean-migration + + git fetch --no-tags --no-recurse-submodules origin \ + refs/heads/codex/clean-migration:${checkpoint_ref} + test "$(git rev-parse "${checkpoint_ref}")" = "${parent_checkpoint}" + + git -C submodules/centerforopenneuroscience.org fetch \ + --no-tags --no-recurse-submodules origin \ + refs/heads/codex/clean-migration:${checkpoint_ref} + test "$(git -C submodules/centerforopenneuroscience.org \ + rev-parse "${checkpoint_ref}")" = "${site_checkpoint}" + + - name: Install the locked Pixi environment + uses: prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347 # v0.10.1 + with: + cache: true + cache-write: ${{ github.event_name != 'pull_request' }} + locked: true + pixi-version: v0.73.0 + + - name: Run focused contracts + run: pixi run test-pages + + - name: Build twice and exercise the static editor + run: pixi run test-pages-browser + + - name: Upload the reviewed static artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + include-hidden-files: true + path: build/pages-preview/orinoco-lite-dev + + deploy: + name: Deploy the reviewed Pages artifact + needs: build + if: >- + github.event_name == 'push' || + (github.event_name == 'workflow_dispatch' && inputs.deploy) + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + contents: read + id-token: write + pages: write + concurrency: + group: con-pages-deployment + cancel-in-progress: true + + steps: + - name: Configure the Pages deployment + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.github/workflows/upstream-pages-trial.yml b/.github/workflows/upstream-pages-trial.yml new file mode 100644 index 0000000..9b656d3 --- /dev/null +++ b/.github/workflows/upstream-pages-trial.yml @@ -0,0 +1,101 @@ +name: Upstream Psychoinformatics Pages trial + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: upstream-psychoinformatics-pages + cancel-in-progress: true + +jobs: + build-and-deploy: + runs-on: ubuntu-24.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + contents: read + pages: write + id-token: write + + steps: + - name: Check out the coordination branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Check out the pinned upstream site only + run: | + git submodule sync -- submodules/www-from-model + git submodule update --init --depth 1 -- submodules/www-from-model + git -C submodules/www-from-model config core.worktree \ + "${GITHUB_WORKSPACE}/submodules/www-from-model" + git -C submodules/www-from-model submodule update --init --depth 1 -- themes/congo + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: 0.12.1 + enable-cache: false + + - name: Install git-annex + run: uv tool install git-annex==10.20260717 + + - name: Hydrate upstream annexed assets + run: | + git -C submodules/www-from-model fetch --depth 1 \ + origin \ + +010ca44f751d2ab60b9d4ad58c5931d1804e3c9e:refs/remotes/origin/git-annex + git -C submodules/www-from-model remote add upstream \ + https://hub.psychoinformatics.de/www/www-from-model.git + git -C submodules/www-from-model fetch --depth 1 \ + upstream \ + +010ca44f751d2ab60b9d4ad58c5931d1804e3c9e:refs/remotes/upstream/git-annex + git -C submodules/www-from-model config --add annex.private true + git -C submodules/www-from-model annex init + git -C submodules/www-from-model annex get . + test -z "$(git -C submodules/www-from-model annex find --not --in=here)" + + - name: Install Hugo Extended + uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3.2.1 + with: + hugo-version: 0.154.5 + extended: true + + - name: Read the Pages deployment URL + id: pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Build the unmodified upstream source + env: + BASE_URL: ${{ steps.pages.outputs.base_url }} + run: | + hugo \ + --minify \ + --source submodules/www-from-model \ + --destination "${RUNNER_TEMP}/upstream-site" \ + --baseURL "${BASE_URL%/}/" + + - name: Adapt and audit the generated Pages artifact + env: + BASE_PATH: ${{ steps.pages.outputs.base_path }} + run: | + python3 tools/adapt_upstream_pages.py \ + "${RUNNER_TEMP}/upstream-site" \ + --base-path "${BASE_PATH:-/}" + python3 tools/adapt_upstream_pages.py \ + "${RUNNER_TEMP}/upstream-site" \ + --base-path "${BASE_PATH:-/}" \ + --check-only + test -f "${RUNNER_TEMP}/upstream-site/index.html" + test -f "${RUNNER_TEMP}/upstream-site/graph.json" + + - name: Upload the Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: ${{ runner.temp }}/upstream-site + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.gitignore b/.gitignore index 76d8577..e3129e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # Local deployment output and credentials. /build/ /dist/ +/node_modules/ *.env !.env.example +__pycache__/ +*.py[cod] diff --git a/.gitmodules b/.gitmodules index e533eae..e2994db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,18 @@ [submodule "submodules/artwork"] path = submodules/artwork - url = https://hub.psychoinformatics.de/orinoco/artwork.git + url = https://github.com/leej3/artwork.git branch = main [submodule "submodules/bids-things"] path = submodules/bids-things - url = https://hub.psychoinformatics.de/orinoco/bids-things.git + url = https://github.com/leej3/bids-things.git branch = main [submodule "submodules/centerforopenneuroscience.org"] path = submodules/centerforopenneuroscience.org - url = https://github.com/con/centerforopenneuroscience.org.git - branch = master + url = https://github.com/leej3/centerforopenneuroscience.org.git + branch = orinoco-lite [submodule "submodules/curatee-kit"] path = submodules/curatee-kit - url = https://hub.psychoinformatics.de/orinoco/curatee-kit.git + url = https://github.com/leej3/curatee-kit.git branch = main [submodule "submodules/dump-research-info"] path = submodules/dump-research-info @@ -20,77 +20,77 @@ branch = agent/git-native-con-metadata [submodule "submodules/dump-things-pyclient"] path = submodules/dump-things-pyclient - url = https://hub.psychoinformatics.de/orinoco/dump-things-pyclient.git + url = https://github.com/leej3/dump-things-pyclient.git branch = master [submodule "submodules/dump-things-service"] path = submodules/dump-things-service - url = https://hub.psychoinformatics.de/orinoco/dump-things-service.git + url = https://github.com/leej3/dump-things-service.git branch = master [submodule "submodules/dump-things-service-mirror"] path = submodules/dump-things-service-mirror - url = https://hub.psychoinformatics.de/orinoco/dump-things-service-mirror.git + url = https://github.com/leej3/dump-things-service-mirror.git branch = master [submodule "submodules/find-things"] path = submodules/find-things - url = https://hub.psychoinformatics.de/orinoco/find-things.git + url = https://github.com/leej3/find-things.git branch = main [submodule "submodules/flatson"] path = submodules/flatson - url = https://hub.psychoinformatics.de/orinoco/flatson.git + url = https://github.com/leej3/flatson.git branch = main [submodule "submodules/flatson-js"] path = submodules/flatson-js - url = https://hub.psychoinformatics.de/orinoco/flatson-js.git + url = https://github.com/leej3/flatson-js.git branch = main [submodule "submodules/flatsonpy"] path = submodules/flatsonpy - url = https://hub.psychoinformatics.de/orinoco/flatsonpy.git + url = https://github.com/leej3/flatsonpy.git branch = main [submodule "submodules/flow"] path = submodules/flow - url = https://hub.psychoinformatics.de/orinoco/flow.git + url = https://github.com/leej3/flow.git branch = main [submodule "submodules/psyinf-pool-files-public"] path = submodules/psyinf-pool-files-public - url = https://hub.psychoinformatics.de/orinoco/psyinf-pool-files-public.git + url = https://github.com/leej3/psyinf-pool-files-public.git branch = main [submodule "submodules/query-things"] path = submodules/query-things - url = https://hub.psychoinformatics.de/orinoco/query-things.git + url = https://github.com/leej3/query-things.git branch = main [submodule "submodules/research-information-ui-assets"] path = submodules/research-information-ui-assets - url = https://hub.psychoinformatics.de/orinoco/research-information-ui-assets.git + url = https://github.com/leej3/research-information-ui-assets.git branch = main [submodule "submodules/shacl-tulip"] path = submodules/shacl-tulip - url = https://hub.psychoinformatics.de/orinoco/shacl-tulip.git - branch = main -[submodule "submodules/shacl-vue"] - path = submodules/shacl-vue - url = https://hub.psychoinformatics.de/orinoco/shacl-vue.git + url = https://github.com/leej3/shacl-tulip.git branch = main [submodule "submodules/some-things"] path = submodules/some-things - url = https://hub.psychoinformatics.de/orinoco/some-things.git + url = https://github.com/leej3/some-things.git branch = main [submodule "submodules/things-enrichment-tools"] path = submodules/things-enrichment-tools - url = https://hub.psychoinformatics.de/orinoco/things-enrichment-tools.git + url = https://github.com/leej3/things-enrichment-tools.git branch = main [submodule "submodules/things-graph-renderer"] path = submodules/things-graph-renderer - url = https://hub.psychoinformatics.de/orinoco/things-graph-renderer.git + url = https://github.com/leej3/things-graph-renderer.git branch = main [submodule "submodules/things-schemas"] path = submodules/things-schemas - url = https://hub.psychoinformatics.de/orinoco/things-schemas.git + url = https://github.com/leej3/things-schemas.git branch = main [submodule "submodules/tools"] path = submodules/tools - url = https://hub.psychoinformatics.de/orinoco/tools.git + url = https://github.com/leej3/tools.git branch = main [submodule "submodules/www-from-model"] path = submodules/www-from-model - url = https://hub.psychoinformatics.de/www/www-from-model.git + url = https://github.com/leej3/www-from-model.git branch = main +[submodule "submodules/pool.psychoinformatics.de-ui"] + path = submodules/pool.psychoinformatics.de-ui + url = https://github.com/leej3/pool.psychoinformatics.de-ui.git + branch = codex/local-deployment diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..e2f47d8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: + - repo: local + hooks: + - id: snapper + name: snapper semantic line breaks + entry: snapper --in-place + language: system + files: '\.(md|markdown|rst|txt)$' diff --git a/AGENTS.md b/AGENTS.md index 6d71637..f98163a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,20 +1,38 @@ # Agent instructions -- Read `docs/orinoco-lite-plan.md` before working and follow its current - milestone and scope. +- Read `docs/orinoco-lite-plan.md`, `docs/clean-migration.md`, `docs/full-con-migration.md`, `docs/milestone-2-acceptance.md`, and `docs/milestone-3.md` before working and follow the active milestone. - Do not begin work listed as deferred or excluded from the current milestone. -- Implement the CON site in `submodules/centerforopenneuroscience.org` on the - `orinoco-lite` branch, which descends from the existing CON `master` branch. - Preserve the legacy branch and tag. +- The active effort is Milestone 3 on the parent `codex/milestone-3` branch. +- Implement site metadata on the successor worktree's `codex/milestone-3` branch, retaining reviewed upstream website commit `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8` until a later reviewed rebase. +- Preserve the accepted parent and site `codex/clean-migration` branches as immutable checkpoints. +Do not amend, rebase, or move them. +- Preserve the accepted parent and site `codex/full-con-migration` branches as immutable Milestone 2 checkpoints. +Do not amend, rebase, or move them. +- Direct upstream ancestry is an intentional exception only for the accepted clean-migration site branch and its full-migration successor. +Do not apply it to `master`, `legacy-site`, `orinoco-lite`, or another CON branch. +- Preserve the legacy CON history, preservation refs, and completed `orinoco-lite` effort unchanged. - Keep `submodules/www-from-model` `main` available to mirror `upstream/main`. -- Adopt upstream code selectively; do not merge or graft the complete - `www-from-model` history into the CON branch. -- Treat `submodules/dump-research-info` as a migration input, not a production - runtime dependency. -- Do not require a continuously running metadata service for builds or deployed - sites. -- Update parent submodule pins deliberately and keep credentials outside every - repository. +- On the successor site branch, use ordinary focused commits for reviewed hand-authored profile and content batches. +Keep one terminal, regenerable projection commit containing generated outputs only; replace or amend it after changes instead of committing generated churn into content batches. +- Keep the CON profile, collections, canonical homepage root, references, and projection isolated from the upstream snapshot. +- Make reviewed YAML in the clean site tree the sole canonical content source. +Milestone 3 explicitly permits repeatable, read-only ingestion from the public CON Zotero API through `submodules/dump-research-info`. +Keep source capture, transformation, review, and site promotion separate, and do not turn that submodule into a production runtime dependency. +- Use the pinned source Things Schema and the `dlthings:*` CURIE contract in `docs/explaining-schema-issues.md`. +Do not use the vendored resolved schema, LinkML trial, or later Things Schemas candidates. +- Do not require a continuously running metadata service for builds or deployed sites. +- Milestone 3 may push exact successor commits to the existing GitHub mirrors, publish one parent `codex/milestone-3` branch, open one draft parent PR against `con/orinoco-lite-dev:main`, and configure that repository's GitHub Pages project preview. +Do not open submodule PRs, change DNS or a custom domain, replace production, publish credentials, or write to Zotero or a public metadata service. +- Hosted editing in this milestone is static and credential-free: it may load public committed metadata and download a review bundle or patch, but it must not create a GitHub token flow, write directly to GitHub, or require a persistent metadata service. +- Record every unresolved human decision in `docs/milestone-3-decisions.md`; do not silently infer publication identity, collection policy, authorship, venue, licensing, or production-cutover semantics. +- Update the parent site gitlink only after the complete local acceptance and rebase checks pass. +Keep credentials outside every repository. + +## User preferences + +- Use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) for every commit subject and body. +- Keep commit subjects and body lines wrapped to approximately 80 columns; avoid long unwrapped commit-message lines. +- Use the Snapper pre-commit hook to auto-format documentation and minimize diffs when editing prose. ## Commit co-authorship @@ -24,4 +42,5 @@ Every commit authored by Codex must include: Co-Authored-By: / ``` -Discover both versions from the active tool and session. Do not guess. +Discover both versions from the active tool and session. +Do not guess. diff --git a/README.md b/README.md index 88aedac..31fa518 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,129 @@ -# Orinoco Lite development +# Full CON migration development workspace -This repository is the development and integration workspace for Orinoco Lite. -It tracks upstream Orinoco components, records architectural decisions, and -coordinates development of a GitHub-native lab website workflow. +This repository coordinates the local-only **full CON migration** for the Center for Open Neuroscience website. +It expands the accepted clean-migration vertical slice into a populated site while keeping canonical content isolated, deterministic, and easy to rebase onto reviewed upstream changes. -Lab websites do not build or deploy from this repository. The first -implementation will live on the `orinoco-lite` branch of the existing -[`centerforopenneuroscience.org`](https://github.com/con/centerforopenneuroscience.org) -repository. That branch continues the established CON history while selectively -adopting useful scaffolding from upstream -[`www-from-model`](https://hub.psychoinformatics.de/www/www-from-model). +The active parent and site branches are `codex/full-con-migration`, using reviewed upstream website commit `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8`. -[`con/www-from-model`](https://github.com/con/www-from-model) remains a clean -GitHub mirror and integration reference for that upstream history. +The parent and site `codex/clean-migration` branches remain immutable accepted checkpoints. +The successor uses focused hand-authored content commits followed by one terminal, regenerable projection commit. + +Legacy CON branches, tags, and the completed `orinoco-lite` prototype remain unchanged. +Nothing in this effort is pushed or deployed. ## Start here -The canonical implementation and handoff document is -[`docs/orinoco-lite-plan.md`](docs/orinoco-lite-plan.md). - -The next milestone is a small, connected CON website preview built on the -`orinoco-lite` branch of `centerforopenneuroscience.org`. It will combine -reviewed metadata migrated from `dump-research-info` with selected content, -assets, and visual identity preserved from the legacy website. - -```mermaid -flowchart LR - U["Upstream Orinoco repositories"] --> D["orinoco-lite-dev"] - W["Upstream www-from-model"] --> M["con/www-from-model mirror"] - M -. "selected code and fixes" .-> I["centerforopenneuroscience.org orinoco-lite branch"] - C["Legacy CON branch, tag, content, and assets"] --> I - R["Reviewed CON research metadata"] --> I - I --> A["GitHub Actions build"] - A --> P["GitHub Pages preview"] - I --> F["Future reusable action and lab template"] +Read the active execution plan and implementation contract before changing the site: + +- [`docs/orinoco-lite-plan.md`](docs/orinoco-lite-plan.md) +- [`docs/full-con-migration.md`](docs/full-con-migration.md) +- [`docs/clean-migration.md`](docs/clean-migration.md) +- [`docs/explaining-schema-issues.md`](docs/explaining-schema-issues.md) + +Install the locked environment and fully initialize every recursive development checkout: + +```console +pixi install --locked +pixi run checkout-submodules +``` + +The main local interfaces are: + +```console +pixi run build # deterministic backend-free CON artifact +pixi run verify-static # require a byte-identical repeat build +pixi run serve-static # CON artifact only, at 127.0.0.1:8767 +pixi run serve # full CON editor/service/static stack +pixi run verify-con-projection # two renders, both matching the Git snapshot +pixi run update-con-assembly # accept reviewed static-assembly inputs +pixi run verify-con-assembly # verify the second committed digest +pixi run build-upstream # explicit German upstream reference build +pixi run serve-upstream # explicit upstream reference server +pixi run test # focused contract tests +pixi run install-browser-tests # one-time Chromium and WebKit setup +pixi run test-browser # managed local browser acceptance +pixi run test-all # unit and browser acceptance +pixi run check-format # repository formatting hooks +``` + +`pixi run build` verifies the committed projection digest, hydrates only the manifest-declared annex assets, and assembles the CON Hugo source in ignored build state. +`pixi run verify-static` also requires a byte-identical repeat build. +The resulting `build/con-site` directory needs no metadata backend. + +Projection updates are explicit: + +```console +pixi run render-con-projection # candidate in ignored build state +pixi run update-con-projection # replace the reviewed Git snapshot +pixi run verify-con-projection # regenerate twice and compare with Git +pixi run update-con-assembly # refresh only the static-input digest +pixi run verify-con-assembly # reject stale static inputs +``` + +While preparing a successor before the parent gitlink moves, a developer may set `CON_SITE_ROOT` to that local site checkout for render/update work. +This is only a temporary workspace override; no absolute successor path belongs in Pixi configuration or committed manifests. +Final build and acceptance run without that override after the parent gitlink points at the reviewed successor tip, and require the site checkout to be clean with exactly one terminal projection snapshot commit. + +A normal build fails closed when relevant canonical records, profile configuration, editorial content, assets, upstream presentation inputs, renderer code, Pixi pins, or component commits change without the corresponding projection or site-assembly refresh. +Metadata changes refresh both digests; editorial, styling, or asset-only changes refresh only the assembly digest. + +Reviewed YAML in the clean site profile is the sole canonical metadata source. +The legacy website and `dump-research-info` are migration evidence only and do not participate in a normal build. + +## Local collection boundary + +The full stack uses four separate collections: + +| Collection | Contents | +| --- | --- | +| `upstream-public` | Cached German public snapshot | +| `upstream-protected` | Local protected counterpart of that snapshot | +| `con-public` | Manifest-declared canonical CON and reference records | +| `con-protected` | CON editor incoming boundary | + +The editor reads and writes only through `con-protected`. +The projection reads only `con-public`. +The cached German records are never seeded into either CON collection. +Tokens, stores, downloaded snapshots, hydrated annex objects, and generated sites stay under ignored `build/` state. + +Curated records in `con-protected` are readable without a token so that an Edit link can populate the concrete SHACL Vue form. +The collection name describes its incoming edit boundary, not confidential curated data. +Submitting a change still requires the ignored token in `build/local-stack/editor-token`, and that token can write only to `con-protected/incoming/local-editor`. + +## Browser acceptance + +Browser dependencies are deliberately separate from the fast unit suite. +The first browser run needs network access to install the exactly locked Playwright package plus its Chromium and WebKit revisions: + +```console +pixi run install-browser-tests +pixi run test-browser ``` -## Core constraints +The browser suite owns ports `8111`, `8122`, `3000`, and `8767` and refuses to reuse an already running stack. +Stop `pixi run serve` before starting it. +Playwright supervises the stack and verifies that all child services stop when the suite ends. + +Chromium and WebKit exercise the real upstream-to-CON same-origin graph-cache transition and the Yaroslav editor link. +A separate Chromium scenario edits a disposable record through SHACL Vue, checks the CON incoming boundary, and cleans the record before and after the test. +It never modifies Yaroslav's real incoming record or records a credential in a URL, trace, screenshot, or video. + +Playwright WebKit is useful Safari-like coverage, but it is not the system Safari browser. +On Linux, Playwright may report missing host browser libraries; this workspace does not install system packages or invoke `--with-deps` automatically. + +## Reproducibility boundary + +Pixi locks Hugo, Python, Dump Things, qri, LinkML, LinkML Runtime, Pydantic, RDFLib, Git Annex `10.20260601`, and their transitive dependencies. +Linux uses the Conda package; macOS ARM uses the pinned Python wheel in the same Pixi environment. +Asset retrieval uses read-only URLs without adding remotes or changing shared worktree configuration. + +The native metadata contract is explicit `dlthings:*` CURIEs against the pinned source Things Schema. +Full-URI type designators, unknown CURIEs, dangling native targets, and generic `AttributeSpecification` relationship bridges are rejected. + +## Scope boundary + +This phase generalizes the proven profile and migrates the legacy-equivalent people, project, editorial, branding, and asset experience. +The broader Zotero collection is deferred. -- Canonical metadata is stored as human-editable YAML in Git. -- Pull requests are the review and publication boundary. -- Dump Things may run ephemerally during CI, but no persistent metadata server - is required for a deployed lab website. -- Generated pages and projections are build artifacts, not canonical records. -- CON-specific content remains downstream; narrow reusable improvements may be - contributed upstream. +It does not publish GitHub Pages, configure pull-request editing, change DNS, update production, or run a persistent hosted metadata service. +The documented rebase drill reviews a candidate upstream range, replays the hand-authored site commits without their terminal generated commit, regenerates the projection, inspects `range-diff`, reruns acceptance, and only then updates the parent gitlink locally. diff --git a/docs/clean-migration.md b/docs/clean-migration.md new file mode 100644 index 0000000..e52cf56 --- /dev/null +++ b/docs/clean-migration.md @@ -0,0 +1,418 @@ +# Clean migration implementation contract + +Status: accepted local-only checkpoint; superseded for active development by [`full-con-migration.md`](full-con-migration.md) + +Reviewed upstream base: `5b401e0c478a4409442b3a8a285bd3efd5d30e05` + +Parent branch: `codex/clean-migration` + +Site branch: `codex/clean-migration` in `submodules/centerforopenneuroscience.org` + +## Purpose + +The clean migration tested a deliberately different history strategy from the completed `orinoco-lite` vertical slice. +The site branch descends directly from the reviewed `www-from-model` upstream commit and carries a minimal CON layer that can be replayed onto later reviewed upstream commits. + +The experiment answered four questions locally: + +1. Can the real CON vertical slice validate and project through the pinned upstream stack while using the source-schema CURIE contract? +2. Can the default service/editor and static interfaces show CON data without mixing in the German snapshot? +3. Can the downstream site remain exactly two commits above upstream? +4. Can those two commits be rebased, regenerated, and reviewed without changing legacy or production state? + +## Exception boundary + +Direct upstream ancestry is allowed for the accepted site branch `codex/clean-migration` and its explicitly authorized `codex/full-con-migration` successor. +It is not a new repository-wide history policy. + +Do not modify, rebase, force-update, merge into, or replace: + +- CON `master`; +- `legacy-site` or its preservation tag; +- the completed `orinoco-lite` branch; +- the clean `www-from-model` mirror branch; or +- any production or deployment ref. + +These direct-upstream branches are local integration branches. +The accepted clean-migration result informs its successor and a later production decision; it does not make that production decision itself. + +## Git topology and two-commit rule + +The site history has this exact shape: + +```text +reviewed www-from-model base: 5b401e0 + | + +-- build-profile commit + | + +-- content-and-projection snapshot commit +``` + +The reviewed upstream base is not counted as a downstream commit. +The branch must have exactly two commits in `5b401e0..codex/clean-migration`. + +### Commit 1: build profile + +The first commit establishes the downstream build and synchronization contract: + +- the account-owned Congo transport URL; +- `UPSTREAM.md` and the two-commit synchronization policy; +- isolated `config/con/` Hugo configuration; +- profile, projection-path, asset-manifest, and source-schema contracts; and +- profile-local storage policy for ordinary Git and annex-backed assets. + +It must not contain canonical CON records, reference records, editorial material, branding payloads, generated entity pages, generated graph data, transient service state, tokens, caches, or built Hugo output. + +### Commit 2: content and projection snapshot + +The second commit contains the reviewed content payload and the deterministic products declared by the first commit: + +- six canonical CON records, including the homepage project, and four provenance-marked reference records; +- editorial content, branding and assets, and migration provenance; +- the deterministic dtc/qri record snapshot; +- generated metadata page bundles; +- graph data produced by the unmodified upstream graph script; and +- a digest tying the snapshot to canonical records, profile configuration, upstream templates, and tool pins. + +Within this commit, canonical/editorial inputs, provenance, and generated projection outputs remain in separate declared paths. +It must not modify upstream layouts, templates, generated German content, refresh workflows, or build policy. +If generation exposes a content defect, fix the content input, regenerate, and amend this same second commit. + +Do not add a third cleanup, conflict-resolution, or test-fix commit. + +## Profile and projection isolation + +The clean-migration profile is a named downstream build context. +It must not overwrite or reinterpret the explicit upstream reference profile. +Generated CON workspaces, service stores, qri caches, and Hugo destinations use profile-specific paths under ignored local build state. + +The normal CON build consumes only: + +- canonical files in the clean site tree; +- the four pinned reference records; and +- explicitly pinned local tool and schema dependencies. + +It must not query the German pool, read the prepared German snapshot, or join records from another repository. + +The upstream snapshot and upstream service/UI remain comparison fixtures. +They require an explicit upstream-named command, argument, collection, or URL. +They are never selected by an unqualified default. + +## Four-collection service contract + +The local Dump Things configuration exposes exactly these logical collection roles: + +| Collection | Allowed records | Consumer | +| --- | --- | --- | +| `upstream-public` | Curated German public snapshot only | Explicit upstream reference reads | +| `upstream-protected` | Local incoming/protected German counterpart only | Explicit upstream editor reference | +| `con-public` | Six canonical CON records and required projection references only | qri and default CON reads | +| `con-protected` | CON curated/incoming editor state only | Default SHACL Vue edit boundary | + +The German snapshot is seeded into `upstream-public` and `upstream-protected`, never either `con-*` collection. + +Canonical CON YAML, including the homepage project, and projection references are loaded into `con-public` and `con-protected`, never either `upstream-*` collection. + +qri caches and projects only `con-public` for the CON site. +The default SHACL Vue configuration reads and writes through `con-protected`. +A curated/public transition may copy an accepted CON edit from `con-protected` to `con-public`; it must never cross into the upstream pair. + +Tests must inspect both collection membership and consumer configuration. +Matching record counts alone are insufficient because the two profiles contain different record sets. + +### Local editor authorization decision + +The word `protected` names the local incoming and editing boundary. +It does not mean that the reviewed curated CON records are confidential. +The default identity for `con-protected` is therefore a dedicated `local_con_reader` with `READ_CURATED` and no write permission. + +The ignored local `local_editor` token is the only non-curator identity with `WRITE_COLLECTION` for `con-protected`. +It may write only to the `local-editor` incoming label. +Anonymous writes and writes to any other collection remain forbidden. + +Static edit links contain the service URL, the generic upstream `dlthings:Thing` node shape, the exact record PID, and `edit=true`. +They never contain a token. +SHACL Vue first reads the record anonymously and then resolves its concrete `xyzri:*` type for the form. +An authenticated write uses the ignored local token through SHACL Vue's local session storage. + +This is a local feedback-loop decision, not a production authentication design. +Patching upstream SHACL Vue, publishing a reusable editor credential, and production authorization remain outside this effort. + +## Static graph cache identity + +The upstream reference artifact and CON artifact may be served from the same loopback origin during local comparison. +They must not reuse an ambiguous `/graph.js` or `/graph.json` browser cache entry. + +Each assembled artifact has one deterministic graph-bundle key. +The key is the SHA-256 digest of a canonical manifest containing the final base-path-adjusted, unversioned `graph.js` digest and the final `graph.json` digest. +The adapter adds the full bundle key to both resource URLs: + +```text +graph.js?v= +graph.json?v= +``` + +Computing the key after base-path adaptation means root and project-path artifacts may have different keys. +The build audit rejects missing, unversioned, stale, or mismatched bundle references and requires adaptation to be idempotent. + +Cache-control headers may improve local feedback, but they are not the correctness mechanism. +The committed static artifact must switch graphs correctly on an ordinary static host without custom response headers or modified upstream templates. + +## Browser acceptance decision + +Browser acceptance belongs to the parent coordination repository because it crosses the static site, Dump Things, and SHACL Vue boundaries. +It uses a root-owned, exactly pinned Playwright dependency and lockfile and does not modify either upstream JavaScript package. + +The fast `pixi run test` command remains browser-free. +Browser dependencies are installed explicitly with `pixi run install-browser-tests`, and the ignored browser suite runs with `pixi run test-browser`. +The full local acceptance command includes both suites. + +Chromium and Playwright WebKit cover the same-origin graph-cache transition and anonymous Yaroslav editor load. +WebKit provides useful Safari-like regression coverage but is not a claim of system Safari equivalence. +A single Chromium scenario performs an authenticated edit of a reserved, disposable person record and proves that it appears only in `con-protected/incoming/local-editor`. +It never edits Yaroslav's real incoming record. + +The authenticated scenario cleans its reserved PID before and after the run. +It disables traces, screenshots, and video, reads ignored token files inside the test process, and never places credentials in URLs, logs, attachments, or assertion output. +All browser traffic remains loopback-only, fixed-port services are not reused, and the test supervisor must clean up every child process even after failure. + +## Source-schema CURIE contract + +The validation path uses the source root schema at: + +```text +submodules/things-schemas/ + src/demo-research-information/unreleased.yaml +``` + +The Things Schemas commit is `d26ea4135e28c25b134c64de1cdc15d15cd2f9f0`. +The service and package pins are those recorded in [`explaining-schema-issues.md`](explaining-schema-issues.md). + +Canonical and generated CON records use these exact native designators: + +```text +dlthings:Association +dlthings:Attribution +dlthings:Generation +dlthings:DOI +dlthings:ISSN +``` + +Do not expand them to full URIs. +Do not validate with the old vendored resolved static schema. +Do not include the LinkML discriminator branch, proposed LinkML composite, or later Things Schemas class-identity candidate. + +Positive tests cover JSON-to-RDF-to-JSON conversion and the live Dump Things endpoint for all five native classes. +Negative tests prove that unsupported full-URI designators fail rather than being silently normalized. + +## Default and reference interfaces + +The unqualified local workflow is CON-focused: + +- the default build validates and projects `con-public`; +- the default full-stack server opens the CON static site and CON editor; +- the default editor points at `con-protected`; and +- the default static-only server serves the generated CON projection. + +The upstream reference workflow remains available for comparison, but every entry point is explicitly labeled upstream. +This includes upstream site builds, static servers, service checks, editor configuration, collection names, and URLs. + +The implementation may retain existing commands through explicit aliases while callers migrate, but it must not leave an ambiguous default that serves German content. +Documentation and startup messages must identify which profile is active. + +All runtime tokens, stores, snapshots, generated workspaces, and service logs remain ignored local state. +No credentials belong in either site commit or the parent gitlink update. + +## Homepage project + +The clean migration follows upstream directly: `xyzrins:.` is a canonical `xyzri:XYZProject` and is the distinguished homepage and project-selection root. +This sixth canonical record represents the CON website and has an explicit native association with the canonical CON organization `ror:04tfhh831`. + +The project and organization are distinct records with distinct purposes. +The homepage project may remain a small placeholder until more CON content is migrated. +The organization remains graph-only until upstream establishes an organization-page convention. + +## Local-only operating boundary + +This effort may read remotes to review upstream commits and retrieve already identified public assets. +It may create local branches, local safety refs, build artifacts, and service state. + +It must not: + +- push any parent, site, mirror, annex, or component ref; +- open or update a pull request; +- enable, configure, or publish GitHub Pages; +- change repository settings, branch protection, or Actions permissions; +- alter DNS, a custom domain, or production hosting; +- write to the public Psychoinformatics pool; +- write to a production annex or metadata service; or +- modify the deployed CON website. + +No successful local test broadens this authorization. + +## Initial construction sequence + +1. Verify the parent is on local branch `codex/clean-migration` and record its starting status. +2. Verify the site worktree is clean and that the reviewed upstream base `5b401e0c478a4409442b3a8a285bd3efd5d30e05` is available locally. +3. Create the site `codex/clean-migration` branch directly at that base without changing legacy-derived refs. +4. Build and amend the build-profile commit until its isolated configuration, manifests, and synchronization policy are complete. +5. Add the reviewed canonical/reference records, editorial material, assets, and provenance; generate the CON projection from a clean local runtime; and create the sole content-and-snapshot commit. +6. Run the complete acceptance suite twice, including a clean regeneration comparison. +7. Inspect the two commits independently and verify each file is in the path and commit declared above. +8. Update the parent site gitlink only after every acceptance criterion passes. +9. Leave all branches and the parent gitlink local. + +## Exact upstream rebase drill + +Perform this drill for every proposed upstream base change. +Resolve and record the three commit IDs before running a mutating command. + +### 1. Review upstream and preserve the old range + +Run this drill from the parent repository. +Resolve the current two-commit range in the site and the proposed base from the clean sibling mirror: + +```bash +clean_site_repo=submodules/centerforopenneuroscience.org +clean_mirror_repo=submodules/www-from-model +clean_old_base="$(git -C "$clean_site_repo" \ + rev-parse codex/clean-migration~2)" +clean_old_tip="$(git -C "$clean_site_repo" \ + rev-parse codex/clean-migration)" +clean_new_base="$(git -C "$clean_mirror_repo" rev-parse main)" +``` + +Before rebasing, verify and preserve the two downstream commits: + +```bash +test "$(git -C "$clean_site_repo" rev-list --count \ + "$clean_old_base..$clean_old_tip")" = 2 +git -C "$clean_site_repo" log --reverse --oneline \ + "$clean_old_base..$clean_old_tip" +git -C "$clean_site_repo" update-ref \ + refs/heads/codex/clean-migration-before-rebase "$clean_old_tip" +``` + +Review `clean_old_base..clean_new_base` in `clean_mirror_repo` before accepting `clean_new_base`. +At minimum inspect the commit log, diffstat, templates, configuration, workflow definitions, generated-content changes, theme gitlink, annex metadata, and any tool or schema references. +Record the reviewed range and relevant changes in profile provenance. + +After review, make that exact mirror commit available to the site through the local sibling repository and reject a moved mirror tip: + +```bash +git -C "$clean_site_repo" fetch ../www-from-model main +test "$(git -C "$clean_site_repo" rev-parse FETCH_HEAD)" = \ + "$clean_new_base" +``` + +The rebased site is the presentation source for configuration, layouts, page templates, and graph code. +The sibling mirror is only the hydration transport for upstream annexed assets and the initialized Congo theme. +Before regeneration, require its `assets`, `static`, and `themes/congo` Git objects to match the candidate base exactly; a mismatch means the sibling pin must be reviewed and updated before continuing. + +### 2. Rebase exactly two commits + +With the site worktree clean: + +```bash +git -C "$clean_site_repo" rebase --onto "$clean_new_base" \ + "$clean_old_base" codex/clean-migration +test "$(git -C "$clean_site_repo" rev-list --count \ + "$clean_new_base..codex/clean-migration")" = 2 +``` + +Resolve build-profile conflicts in the first commit and content/projection conflicts in the second. +Keep content inputs and generated output in their isolated paths. +Do not add a third commit. + +### 3. Regenerate the projection + +Run the profile's locked validation and generation entry point from clean ignored runtime state. +Verify that it reads `con-public`, not `upstream-public`. + +Replace the tracked generated projection with the new result, review the manifest, and amend the second commit: + +```bash +git -C "$clean_site_repo" commit --amend --no-edit +test "$(git -C "$clean_site_repo" rev-list --count \ + "$clean_new_base..codex/clean-migration")" = 2 +``` + +If regeneration requires a content change, amend that input and regenerated output together into the second commit. +If it requires a build-profile or policy change, amend the first commit, rerun generation, and amend the second commit again. + +### 4. Compare the downstream ranges + +Use the preserved local ref to review semantic changes in both commits: + +```bash +git -C "$clean_site_repo" range-diff \ + "$clean_old_base..refs/heads/codex/clean-migration-before-rebase" \ + "$clean_new_base..codex/clean-migration" +``` + +The range-diff must show one build-profile commit and one content-and-projection snapshot commit. +Review unexpected profile changes as contract changes and expected generated churn through the projection manifest. + +### 5. Run acceptance + +Run the locked unit, source-schema CURIE, live service, four-collection, projection, repeat-generation, link, graph, base-path, and local browser checks. +Confirm the default interfaces are CON and upstream references require explicit selection. + +Also verify that legacy-derived refs and all remote state are unchanged. +Do not treat conflict-free rebase or a successful Hugo build as sufficient acceptance. + +### 6. Update the parent gitlink deliberately + +Only after acceptance, return to the parent repository and inspect the proposed site gitlink change with submodule log output. +The parent change must point to the accepted two-commit site tip and must not move another submodule. + +Record the old base, new base, old tip, new tip, range-diff review, projection manifest, and acceptance result alongside the deliberate gitlink update. +Do not push either repository. + +Keep the before-rebase safety ref until the range-diff and parent gitlink review are complete. +Its later local cleanup is not part of the rebase itself. + +## Acceptance matrix + +| Boundary | Required result | +| --- | --- | +| Site history | Exactly two downstream commits above the reviewed base | +| Path isolation | Build contract, content inputs, and generated outputs remain in their declared isolated paths | +| Presentation source | Config, layouts, templates, and graph code come from the rebased site ancestry | +| Annex transport | Sibling asset/static/theme Git objects match that ancestry exactly | +| Source schema | Pinned source YAML, never the vendored resolved schema | +| Native types | Five exact `dlthings:*` CURIE fixtures pass conversion and live validation | +| Negative types | Full-URI fixtures fail closed | +| CON data | Present only in `con-public` and `con-protected` | +| German snapshot | Present only in `upstream-public` and `upstream-protected` | +| qri | Reads only `con-public` for the CON projection | +| Editor | Default SHACL Vue boundary is `con-protected` | +| Static default | Default build/server presents CON content | +| Upstream references | Available only through explicit upstream selection | +| Projection | Repeat generation is deterministic and manifest-reviewed | +| Site integrity | Pages, links, graph, and base paths pass their audits | +| Runtime | No persistent service is needed for the static result | +| Preserved history | Legacy and `orinoco-lite` refs are unchanged | +| External state | No push, Pages, DNS, domain, or production change | + +## Deferred work + +Do not expand this effort to include: + +- complete CON metadata or editorial migration; +- production deployment or branch replacement; +- GitHub Pages or pull-request preview workflows; +- remote editing, authentication, or authorization design; +- support for full-URI designators; +- experimental LinkML or Things Schemas candidates; +- a generalized root/profile or detailed ROR design; +- a permanent metadata service; +- durable custody for every upstream annex object; +- broad visual redesign; +- action/template extraction; +- published RDF or JSONL interfaces; +- secondary projections; or +- upstream contribution work. + +The clean migration ended with a locally accepted two-commit site branch and a deliberately reviewed local parent gitlink. +Its former deferred full-content work is governed by [`full-con-migration.md`](full-con-migration.md); it remains outside this frozen checkpoint. diff --git a/docs/explaining-schema-issues.md b/docs/explaining-schema-issues.md new file mode 100644 index 0000000..a2af866 --- /dev/null +++ b/docs/explaining-schema-issues.md @@ -0,0 +1,197 @@ +# CURIE-only schema contract for the clean migration + +Status: verified implementation guide + +Date: 2026-08-11 + +## Conclusion + +The clean migration does not need the LinkML discriminator trial or the later Things Schemas identity candidate. + +The verified path uses the checked-out source schema at commit `d26ea413`, the released LinkML stack pinned below, and explicit `dlthings:*` CURIEs in record type designators. +With that combination, native `Association`, `Attribution`, `Generation`, `DOI`, and `ISSN` values pass JSON to RDF to JSON conversion and live Dump Things validation. + +Equivalent full-URI spellings are not supported by this contract. +Records must retain the CURIE spelling expected by the source schema and the downstream qri and template code. + +## Verified runtime + +The verified environment is: + +| Component | Exact selection | +| --- | --- | +| Things Schemas | `d26ea4135e28c25b134c64de1cdc15d15cd2f9f0` | +| Root schema | `src/demo-research-information/unreleased.yaml` | +| Dump Things | `9f101d97c7f15d491f602db5a9c33ad9a19ad8bf` | +| Dump Things release | `6.3.6` | +| LinkML | `1.11.1` | +| LinkML Runtime | `1.11.1` | +| Pydantic | `2.13.4` | +| RDFLib | `7.6.0` | + +The complete root-schema path is: + +```text +submodules/things-schemas/ + src/demo-research-information/unreleased.yaml +``` + +The parent Pixi environment installs Dump Things from its pinned local submodule and locks the Python dependencies. +The service reads the schema YAML directly from the pinned Things Schemas checkout. + +Things Schemas is not installed as a Python development package in this path. +Its install-time `tools/patch_linkml` process therefore does not modify the Pixi environment. +The Dump Things source and its runtime behavior at the pinned service commit are part of the verified combination. + +## Record spelling contract + +Use these exact values for `schema_type`: + +| Native class | Required designator | +| --- | --- | +| Association | `dlthings:Association` | +| Attribution | `dlthings:Attribution` | +| Generation | `dlthings:Generation` | +| DOI | `dlthings:DOI` | +| ISSN | `dlthings:ISSN` | + +For example: + +```yaml +schema_type: dlthings:Association +``` + +Do not expand that value before validation, conversion, storage, or projection. + +The source modules use `default_prefix: dlthings`. +The `dlthings` prefix maps to this namespace: + +```text +https://concepts.datalad.org/s/things/v2/ +``` + +The component files are named `things-prov/v1` and `things-publications/v1`. +Those module names, the root schema's `UNRELEASED` label, and the `/things/v2/` identifier namespace describe different layers. +They do not require a type designator to use a module-derived full URI. + +## What was verified + +Positive fixtures for all five native classes were exercised against the source schema and pinned runtime. +They pass both of the publication-relevant boundaries: + +1. JSON is loaded, converted to RDF, and converted back to JSON without losing the intended native type. +2. The records pass validation through the live local Dump Things collection endpoint using the same pinned schema and package set. + +This is the relevant evidence for the clean migration. +A Hugo build of already generated Markdown is not schema evidence because it does not load LinkML or Dump Things. + +The downstream publication tools also favor one stable lexical form. qri class selection, graph dispatch, and existing templates compare type designators as strings. +Using the source-schema CURIEs from ingestion through projection avoids a second normalization layer. + +## Full URIs remain unsupported + +The CURIE contract does not promise that a full URI is interchangeable with its compact spelling. +For example, this value is outside the supported input contract: + +```text +https://concepts.datalad.org/s/things/v2/Association +``` + +Module-derived values are also outside the contract: + +```text +https://concepts.datalad.org/s/things-prov/v1/Association +https://concepts.datalad.org/s/things-publications/v1/DOI +``` + +Full-URI designators continue to fail the generated-model path at the pinned versions. +The clean migration should reject them rather than rewrite them implicitly. +An input migration should convert a known supported value to the required CURIE explicitly and review that semantic change. + +Supporting full URIs later would be a separate compatibility feature. +It would need its own LinkML, Dump Things, qri, graph, and template tests. +It is not a prerequisite for the clean migration. + +## Why the earlier diagnosis was wrong + +The earlier CON candidate did not exercise the source schema above. +It used this vendored resolved schema: + +```text +submodules/centerforopenneuroscience.org/ + metadata/schema/demo-research-information.static.yaml +``` + +That generated artifact flattened the imports and added explicit expanded `class_uri` values. +For the affected classes it used module-derived identities such as: + +```text +https://concepts.datalad.org/s/things-prov/v1/Association +https://concepts.datalad.org/s/things-prov/v1/Attribution +https://concepts.datalad.org/s/things-prov/v1/Generation +https://concepts.datalad.org/s/things-publications/v1/DOI +https://concepts.datalad.org/s/things-publications/v1/ISSN +``` + +Those expanded identities changed the generated type-designator behavior. +The resulting failures demonstrated a problem with the resolved artifact and full-URI path, not a failure of the source schema's `dlthings:*` CURIE path. + +The previous synthetic reproducer also declared an explicit full class URI. +It remains useful evidence that the full-URI path is unsupported, but it does not model the verified CURIE-only publication contract. + +Claims that no single designator works at these released package versions are therefore superseded. +The verified source-schema CURIEs work without the proposed LinkML changes. + +## Explicitly excluded work + +Do not merge, cherry-pick, install, or otherwise depend on the separate LinkML discriminator trial. +This exclusion includes every commit from the parent trial branch: + +```text +codex/linkml-discriminator-trial +``` + +It also excludes any local composite of proposed LinkML pull-request heads. + +The clean migration does not use proposed LinkML changes for: + +- full-URI subclass dispatch; +- CURIE/full-URI equivalence in generated JSON Schema; +- URI compaction fallback; or +- cross-generator compliance for explicit full class URIs. + +Do not adopt the later Things Schemas candidate that adds explicit `class_uri` declarations for the five classes. +In particular, commit `33604b1a` and its prerequisite candidate commits are outside the clean-migration dependency set. + +The source schema remains pinned at `d26ea413`. +The clean migration must not advance that pin indirectly while updating another submodule or regenerating a resolved schema. + +## Build and test rules + +The implementation should preserve these rules: + +1. Read the source root schema from the pinned Things Schemas checkout. +2. Store the five native designators as the exact `dlthings:*` CURIEs above. +3. Do not use the vendored resolved static schema as a validation input. +4. Do not normalize supported CURIEs to full URIs between service and qri. +5. Keep Dump Things, LinkML, LinkML Runtime, Pydantic, and RDFLib locked as one tested environment. +6. Retain positive JSON-to-RDF-to-JSON and live-validation fixtures for all five classes. +7. Retain negative fixtures showing that full-URI designators are outside the supported contract. +8. Re-run the entire fixture set before changing any schema or package pin. + +The static deployment remains a generated projection and requires no continuously running metadata service. +Dump Things may start ephemerally for validation and projection, then stop before the Hugo artifact is deployed. + +## Evidence boundaries + +This guide establishes the tested local clean-migration contract. +It does not claim that: + +- full URIs work as type designators; +- CURIE and full-URI spellings are accepted interchangeably; +- the German production pool uses this exact package set; +- a resolved static schema is equivalent to its source imports; +- the excluded LinkML or Things Schemas candidates are incorrect for other use cases; or +- a successful static Hugo build proves metadata validation. + +Within those boundaries, the implementation choice is simple: use the pinned source schema, keep explicit `dlthings:*` CURIEs, and leave the experimental LinkML and schema-remediation branches out of the clean migration. diff --git a/docs/full-con-migration.md b/docs/full-con-migration.md new file mode 100644 index 0000000..83851fa --- /dev/null +++ b/docs/full-con-migration.md @@ -0,0 +1,242 @@ +# Full CON migration implementation contract + +Status: active, local-only successor phase + +Reviewed upstream base: `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8` + +Parent branch: `codex/full-con-migration` + +Site branch: `codex/full-con-migration` in `submodules/centerforopenneuroscience.org` + +Accepted clean-migration checkpoint: + +- parent `f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d`; and +- site `a122e506de9e4a13473edbe8d74a950d74032a16`. + +## Purpose + +The clean migration proved that CON metadata can use the upstream storage, projection, presentation, editor, and static-build conventions without mixing in the German snapshot or requiring a persistent metadata service. +It also proved that a small CON layer can be reviewed and replayed onto upstream. + +The full migration turns that successful vertical slice into the complete CON website. +It retains the isolated profile and deterministic static deployment, generalizes the remaining slice-specific contracts, and migrates the reviewed public content in coherent batches. + +This phase is not a production cutover. +All work, acceptance, and upstream synchronization remain local. + +## Accepted checkpoint and successor boundary + +The parent and site `codex/clean-migration` branches are accepted checkpoints. +Do not amend, rebase, delete, or move them. +They remain the compact evidence that the architecture works with six canonical records, including the ordinary `xyzrins:.` project root. + +The active parent and site branches are both named `codex/full-con-migration`. +The site successor retains direct upstream ancestry as the second explicit exception to the legacy-derived branch policy. +That exception does not apply to `master`, `legacy-site`, `orinoco-lite`, or any production ref. + +The successor's reviewed upstream base is `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8`. +The range after the clean migration's `5b401e0` base contains one reviewed change: + +- `ci: use deposit-changes from orinoco/flow`, which changes only `.forgejo/workflows/register-depictions.yaml` from a path reference to a URL. + +It does not change templates, content generation, Hugo configuration, graph rendering, assets, or the Congo theme. +The full acceptance suite is still required after replaying the CON layer. + +## Site history policy + +The two-commit rule belongs only to the accepted clean-migration checkpoint. +The full migration uses normal Conventional Commits so content review remains legible as the site grows. + +The site history has three conceptual parts: + +1. the two accepted CON foundation commits, replayed onto the reviewed upstream base; +2. focused, reviewed commits containing hand-authored profile changes and coherent content batches; and +3. one terminal `chore(projection): refresh the full CON snapshot` commit with generated projection outputs and their digests. + +The terminal projection commit contains no hand-authored canonical YAML, editorial prose, migration decisions, profile contracts, or source assets. +It may be dropped before a rebase and recreated afterward, or amended after a content batch. +Generated churn must not obscure the review of hand-authored content. + +Parent tooling, tests, policy, and the deliberate site gitlink use focused ordinary commits. +The parent gitlink moves only after site acceptance passes. + +## Canonical content and migration evidence + +Reviewed YAML under the isolated CON profile in the site repository is the sole canonical metadata source. +Editorial Markdown, profile configuration, and declared source assets in that same tree are the sole static-site inputs. + +The two migration evidence sources have complementary roles: + +| Evidence source | Permitted use | +| --- | --- | +| Legacy CON site history | Public roster and project selection, editorial voice, navigation intent, ordering, branding, imagery, and provenance | +| `dump-research-info` | Candidate structured fields, identity reconciliation, relationships, vocabulary requirements, and source provenance | + +Neither evidence repository is a normal build-time dependency. +Import and reconciliation tools may read them explicitly while preparing a candidate batch, but the reviewed result must be copied into clean-site YAML or editorial paths before it can enter the build. + +Every migrated record has a provenance entry recording its evidence paths, identity decisions, material transformations, unresolved fields, and review status. +A later change to either evidence source never changes the canonical site implicitly. + +Do not copy the earlier generic relationship overlay, URL-based provisional PIDs, missing nested discriminators, vendored schema, generic projector, or custom renderer into the clean site. +Reconcile useful facts into the native upstream contract instead. + +## Milestone 1: generalize the proven contracts + +Before bulk content migration, replace vertical-slice constants with one executable profile manifest while preserving the accepted six-record slice as the regression fixture. + +The manifest must declare or derive: + +- canonical record paths and the distinguished `xyzrins:.` root; +- renderable classes, route families, page visibility, and navigation order; +- supporting reference-data closure and provenance; +- expected native relationships and allowed dangling-target policy; +- asset ownership, hydration source, availability, fallback, and destination; +- editor-link eligibility and collection boundaries; and +- graph, route, and content invariants used by build and browser acceptance. + +Projection, stack preparation, graph audits, editor-link checks, asset hydration, and tests must consume that contract rather than restating exact lists for six records, four references, seven edges, five pages, or one portrait. +Yaroslav, DataLad, the publication, the instrument, the organization, and `xyzrins:.` remain representative smoke assertions rather than the complete inventory. + +Split invalidation into two reviewed digests: + +- a metadata-projection digest covering canonical/reference YAML, projection rules, schema and tool pins, and renderer inputs; and +- a site-assembly digest covering the committed projection, editorial content, Hugo profile, upstream templates, assets, and static adaptation. + +An editorial-only change must require a new static artifact but not a metadata reprojection. +A metadata or projection-contract change must regenerate both. +Both paths fail closed on stale committed products. + +## Milestone 2: restore legacy-equivalent public coverage + +After the generalized six-record suite passes, migrate the legacy site's public experience in reviewable batches. +The initial evidence inventory has 33 visible people and 23 featured projects. +Treat those counts as a reconciliation baseline, not permission to manufacture records; any reviewed exclusion, addition, or merge must be recorded in provenance. + +The first coverage target includes: + +- the visible people roster, its public ordering, reviewed biographies, roles, links, and available portraits; +- the featured project roster, its public ordering, descriptions, links, native associations, and available artwork; +- the homepage, navigation, contact/support material, and other legacy editorial pages needed to understand the organization and move through the site; and +- CON branding and reviewed imagery with explicit provenance and fallback behavior. + +`xyzrins:.` remains the canonical distinguished project root and homepage. +It is not counted as a featured research project unless a separate content review chooses to present it that way. +The CON ROR organization remains a graph record without a dedicated page until the upstream presentation defines an organization route or this phase records an explicit downstream decision. + +Legacy-equivalent means equivalent public information, navigation intent, and recognizable CON identity. +It does not require pixel-level reproduction of the legacy theme. +Prefer upstream layouts and conventions, profile-local editorial content, configuration, assets, and narrowly scoped styling over a downstream template fork. + +## Content-batch policy + +Each content batch must be small enough to review as a semantic unit. +Suitable batches include a reconciled group of people, a connected project cluster, or one editorial section with its declared assets. + +A batch is complete only when it includes: + +- canonical YAML with stable CURIE/PID decisions; +- native relationship records and the required reference closure; +- editorial and asset inputs owned by that batch; +- migration provenance and explicitly deferred fields; +- validation, route, graph, link, and collection expectations; and +- a regenerated terminal projection commit after the hand-authored commit has been reviewed. + +Use the pinned source Things Schema and exact `dlthings:*` CURIE designators. +Typed DOI and ISSN values remain identifiers. +Relationships use native `Association`, `Attribution`, and `Generation` records rather than generic `AttributeSpecification` bridges. + +When identity, membership status, project visibility, asset licensing, or public wording is ambiguous, record the candidate and evidence in the migration ledger and continue with unambiguous records. +Do not invent public semantics merely to satisfy an expected count. +Surface the unresolved decision before publishing or treating that batch as parity-complete. + +## Asset policy + +Every presented asset must be an ordinary verified Git blob or a manifest entry with a retrievable read-only annex source and expected key/digest. +The manifest distinguishes available, intentionally omitted, and unavailable assets and declares the fallback for the latter two states. + +Never present an annex pointer text file as an image, copy a large annex object into ordinary Git merely to simplify the migration, merge unrelated annex histories, or add a writable remote. +Preserve source path/key provenance and any known licensing or attribution information. + +Missing imagery does not block migration of otherwise reviewed metadata. +It uses a deliberate neutral fallback and remains visible in the migration ledger for later custody work. + +## Build, service, and deployment contracts + +The proven clean-migration boundaries remain active: + +- the static artifact builds only from committed clean-site inputs and the pinned local toolchain; +- `con-public` is the sole CON projection source; +- `con-protected` is the local editor boundary with anonymous curated reads and token-limited incoming writes; +- the German snapshot remains isolated in `upstream-public` and `upstream-protected`; +- unqualified commands present CON, while upstream references are explicit; +- graph resources share an audited content-derived cache identity; +- root and project-path static builds remain deterministic; and +- no metadata service is required after static generation. + +As content grows, collection and graph assertions derive their expected inventory from the reviewed profile manifest. +They continue to prove exact membership, native target integrity, and the absence of German records rather than relying on hard-coded totals. + +## Acceptance + +Milestone 1 is accepted when the generalized contract reproduces the accepted vertical slice byte-for-byte where inputs are unchanged and all unit, projection, static, collection, and Playwright checks pass. + +Each later content batch must pass: + +- source-schema validation and JSON-to-RDF-to-JSON round trips for its native types; +- unknown-CURIE, full-URI designator, and dangling-target negative tests; +- deterministic projection verification and two byte-identical static builds; +- exact manifest-derived collection and graph membership checks; +- expected root and project-path routes, links, branding, and assets; +- absence of German entity routes and graph nodes from the CON artifact; +- Chromium and WebKit navigation, graph, and representative editor checks; +- the disposable authenticated-editor boundary test; and +- process, token, incoming-probe, and temporary-state cleanup. + +The full legacy-coverage milestone also requires a reviewed reconciliation report for the people roster, featured projects, editorial pages, and assets. +The report explains every difference from the evidence inventory. + +## Upstream synchronization + +Upstream remains a reviewed input, never an automatic merge target. +For each candidate base: + +1. preserve the current full-migration tip and record the candidate upstream commit; +2. review the upstream range, especially templates, projection behavior, workflows, theme and annex changes; +3. drop the terminal generated projection commit from the replay range; +4. rebase the accepted foundation and ordinary hand-authored commits onto the exact reviewed base; +5. regenerate one terminal projection commit from clean ignored state; +6. inspect `range-diff`, the two digests, and expected generated churn; +7. run complete local acceptance; and +8. update the parent gitlink deliberately only after acceptance. + +Never rebase or move the accepted `codex/clean-migration` checkpoint as part of this drill. +If an upstream change requires compatibility code, first attempt to adapt the profile or content to the new convention. +Record unavoidable downstream divergence as an explicit contract decision. + +## Local-only operating boundary + +Remote reads may be used to review upstream and retrieve already identified public assets. +Local branches, safety refs, generated state, and test services are allowed. + +This phase must not push refs, open or update pull requests, publish Pages, modify repository settings, alter DNS or production hosting, write to the public Psychoinformatics pool, or place credentials in a repository. +A successful acceptance run does not broaden this authorization. + +## Explicitly deferred work + +The following work remains outside the active milestones: + +- importing or publishing the broader Zotero collection; +- bulk publication migration beyond records required by the reviewed legacy people/project experience; +- GitHub Pages, preview deployments, production cutover, DNS, redirects, and custom domains; +- pull-request-based editing, GitHub Apps, OAuth, hosted editor authentication, and branch-protection design; +- a persistent hosted metadata service; +- support for full-URI type designators or experimental schema/LinkML branches; +- grants, CVs, annual reports, and secondary projections; +- pixel-level legacy-theme reproduction or a broad upstream template fork; +- durable custody for every historical annex object; +- a separate metadata repository or published RDF/JSONL interface; and +- upstream contribution work unrelated to a migration blocker. + +Revisit Zotero only after the legacy-equivalent people, project, editorial, and asset reconciliation is accepted. +Revisit deployment and pull-request editing as separate, explicitly authorized phases after the static content migration is stable. diff --git a/docs/milestone-1-design-review.md b/docs/milestone-1-design-review.md index d697408..09ac2ee 100644 --- a/docs/milestone-1-design-review.md +++ b/docs/milestone-1-design-review.md @@ -2,11 +2,9 @@ Status: resolved in the revised Milestone 1 prototype -Original candidate reviewed: `centerforopenneuroscience.org` at -`2c4f7e5a19d8ade7aee25ef1e8dc786bfdb3a577` +Original candidate reviewed: `centerforopenneuroscience.org` at `2c4f7e5a19d8ade7aee25ef1e8dc786bfdb3a577` -Revised candidate: `centerforopenneuroscience.org` at -`2621231d27b70fb425107a132159f7a9e0d99cda` +Revised candidate: `centerforopenneuroscience.org` at `2621231d27b70fb425107a132159f7a9e0d99cda` Reviewed `www-from-model`: `6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce` @@ -14,83 +12,52 @@ Date: 2026-07-31 ## Resolution implemented -The revised prototype now uses the functional website layer that was missing -from the original candidate. It selectively restores the reviewed -`www-from-model` taxonomy and term layouts, metadata-derived class lists, -filter controls, forward relationship terms, Hugo-derived reverse backlinks, -qri inlining, and the pinned Things graph renderer. The same validated record -stream generates every entity page, list membership, relationship panel, and -graph node and edge. Adding a connected sixth record requires no route, -template, index, or Python inventory change. +The revised prototype now uses the functional website layer that was missing from the original candidate. +It selectively restores the reviewed `www-from-model` taxonomy and term layouts, metadata-derived class lists, filter controls, forward relationship terms, Hugo-derived reverse backlinks, qri inlining, and the pinned Things graph renderer. +The same validated record stream generates every entity page, list membership, relationship panel, and graph node and edge. +Adding a connected sixth record requires no route, template, index, or Python inventory change. The implementation also resolves the operational concerns raised below: -- `enrich_projection.py`, `validate_records.py`, the individual gate scripts, - and `install-hugo.sh` were removed; -- the exact YAML streams now pass directly through upstream `dtc` and an - ephemeral Dump Things service; +- `enrich_projection.py`, `validate_records.py`, the individual gate scripts, and `install-hugo.sh` were removed; +- the exact YAML streams now pass directly through upstream `dtc` and an ephemeral Dump Things service; - Pixi is the sole locked Python, Hugo, Node, and Linux CI environment; -- entity `_index.md` bundles are generated, while checked-in collection - `_index.md` files contain only editorial introductions and presentation - settings; -- Yaroslav's portrait remains the pre-existing git-annex object and CI - retrieves and verifies it from `datasets.datalad.org`; and -- the graph is a compact side panel by default, with an accessible link view - and responsive stacking on narrow screens. +- entity `_index.md` bundles are generated, while checked-in collection `_index.md` files contain only editorial introductions and presentation settings; +- Yaroslav's portrait remains the pre-existing git-annex object and CI retrieves and verifies it from `datasets.datalad.org`; and +- the graph is a compact side panel by default, with an accessible link view and responsive stacking on narrow screens. The native type-discriminator incompatibility is the one accepted exception. -It is pinned, reproduced, and documented separately in the site repository's -`docs/upstream-schema-discriminator-issue.md`, including the tested options, -trade-offs, pin-update discipline, and removal condition. The current bridge -is one generic projection with retained raw assertions; it contains no CON -PID, label, route, or asset table. Per the design decision, no local or -provisional upstream patch is being accumulated while the compatible upstream -tuple is unresolved. Native qualified roles and typed DOI/ISSN records remain -explicitly deferred. - -The full acceptance suite passes deterministic repeat builds, direct upstream -schema rejection, dangling-target rejection for arbitrary PID syntax, exact -metadata/page authority, base-path links, backlinks, graph consistency, and a -metadata-only sixth-record extension. Visual theming and broader organization -are intentionally left for a later focused design pass. +It is pinned, reproduced, and documented separately in the site repository's `docs/upstream-schema-discriminator-issue.md`, including the tested options, trade-offs, pin-update discipline, and removal condition. +The current bridge is one generic projection with retained raw assertions; it contains no CON PID, label, route, or asset table. +Per the design decision, no local or provisional upstream patch is being accumulated while the compatible upstream tuple is unresolved. +Native qualified roles and typed DOI/ISSN records remain explicitly deferred. + +The full acceptance suite passes deterministic repeat builds, direct upstream schema rejection, dangling-target rejection for arbitrary PID syntax, exact metadata/page authority, base-path links, backlinks, graph consistency, and a metadata-only sixth-record extension. +Visual theming and broader organization are intentionally left for a later focused design pass. ## Original conclusion (superseded by the resolution above) -The remainder of this document records the diagnosis and options that led to -the revised prototype. Statements about what the original `2c4f7e5` candidate -did or lacked are historical and do not describe `2621231`. +The remainder of this document records the diagnosis and options that led to the revised prototype. +Statements about what the original `2c4f7e5` candidate did or lacked are historical and do not describe `2621231`. -Milestone 1 proves that five repository-resident CON records can be validated -with an ephemeral Dump Things service and rendered reproducibly by qri and -Hugo. It does **not** yet prove the intended metadata-driven website -architecture. +Milestone 1 proves that five repository-resident CON records can be validated with an ephemeral Dump Things service and rendered reproducibly by qri and Hugo. +It does **not** yet prove the intended metadata-driven website architecture. -The candidate retained the upstream command-line spine—Dump Things, `dtc`, qri, -Jinja, Hugo, and Congo—but replaced most of the upstream behavior that made the -site navigable as a metadata graph. In particular, it omitted relationship -injection and inlining, taxonomy-derived class lists and backlinks, the graph -projection, and the graph interface. Those behaviors were replaced by a -five-record Python view adapter and hand-authored class indexes. +The candidate retained the upstream command-line spine—Dump Things, `dtc`, qri, Jinja, Hugo, and Congo—but replaced most of the upstream behavior that made the site navigable as a metadata graph. +In particular, it omitted relationship injection and inlining, taxonomy-derived class lists and backlinks, the graph projection, and the graph interface. +Those behaviors were replaced by a five-record Python view adapter and hand-authored class indexes. -That trade was acceptable for diagnosing whether the components could run -together, but it is not an acceptable baseline for the complete migration or a -reusable lab template. Draft PR 84 should therefore be treated as a technical -spike. Milestone 2 and production cutover should wait until the metadata, -navigation, compatibility, and asset decisions below are resolved. +That trade was acceptable for diagnosing whether the components could run together, but it is not an acceptable baseline for the complete migration or a reusable lab template. +Draft PR 84 should therefore be treated as a technical spike. +Milestone 2 and production cutover should wait until the metadata, navigation, compatibility, and asset decisions below are resolved. The recommended direction is: -1. restore native qualified relationships and typed identifiers through the - tested narrow LinkML/LinkML-runtime fix and exact component pins; -2. restore a base-path-safe subset of `www-from-model`'s qri relationship, - taxonomy, backlink, and graph pipeline; -3. use legacy CON as the source of identity, editorial content, URL - compatibility, and visual design—not as a reason to replace the upstream - information architecture; -4. fetch required annexed assets from an availability-tested remote during the - build; and -5. use Pixi to pin Hugo and the Python toolchain, eliminating the custom Hugo - downloader. +1. restore native qualified relationships and typed identifiers through the tested narrow LinkML/LinkML-runtime fix and exact component pins; +2. restore a base-path-safe subset of `www-from-model`'s qri relationship, taxonomy, backlink, and graph pipeline; +3. use legacy CON as the source of identity, editorial content, URL compatibility, and visual design—not as a reason to replace the upstream information architecture; +4. fetch required annexed assets from an availability-tested remote during the build; and +5. use Pixi to pin Hugo and the Python toolchain, eliminating the custom Hugo downloader. ## What the candidate currently proves @@ -106,43 +73,32 @@ five canonical YAML files -> fork-only GitHub Pages preview ``` -It demonstrates repository-contained input, deterministic output, failure on -invalid records, no persistent metadata service, and no production deployment -change. Those are useful results and should be retained. +It demonstrates repository-contained input, deterministic output, failure on invalid records, no persistent metadata service, and no production deployment change. +Those are useful results and should be retained. The missing product invariant is stronger: -> Adding a valid, connected record should produce its page, place it in the -> appropriate class navigation, create its forward and reverse navigation, -> and add its graph node and edges without editing Python, templates, class -> indexes, or acceptance constants. +> Adding a valid, connected record should produce its page, place it in the > appropriate class navigation, create its forward and reverse navigation, > and add its graph node and edges without editing Python, templates, class > indexes, or acceptance constants. -The current candidate does not satisfy that invariant. A sixth record fails -unless it is added to `scripts/enrich_projection.py`; it would not -automatically appear in a class index; and no graph artifact is generated. +The current candidate does not satisfy that invariant. +A sixth record fails unless it is added to `scripts/enrich_projection.py`; it would not automatically appear in a class index; and no graph artifact is generated. ### Corrections to the current progress report Three statements should be revised after the design is agreed: -- “Milestone 1 complete” is accurate only against the original literal exit - criteria. A clearer status is “technical spike complete; design acceptance - pending.” -- The report says Yaroslav's annex payload had no available copies. Direct - remote probing shows that `datasets.datalad.org` has the key. -- The provenance inventory identifies copied and adapted files, but it does - not make the functional omission of taxonomy navigation, backlinks, graph - UI, and add-record behavior sufficiently clear. +- “Milestone 1 complete” is accurate only against the original literal exit criteria. +A clearer status is “technical spike complete; design acceptance pending.” +- The report says Yaroslav's annex payload had no available copies. +Direct remote probing shows that `datasets.datalad.org` has the key. +- The provenance inventory identifies copied and adapted files, but it does not make the functional omission of taxonomy navigation, backlinks, graph UI, and add-record behavior sufficiently clear. -The successful build evidence remains valid; these corrections change how the -result should be interpreted, not whether that candidate commit built. +The successful build evidence remains valid; these corrections change how the result should be interpreted, not whether that candidate commit built. ## Why there are so many scripts -There are ten files and 1,233 lines under `scripts/`. The count is inflated -because ordinary build orchestration, upstream API adaptation, a temporary -schema workaround, site projection, and milestone tests all share one -directory and several tests run inside every build. +There are ten files and 1,233 lines under `scripts/`. +The count is inflated because ordinary build orchestration, upstream API adaptation, a temporary schema workaround, site projection, and milestone tests all share one directory and several tests run inside every build. | Script | Why it exists | Assessment | Intended fate | | --- | --- | --- | --- | @@ -157,64 +113,49 @@ directory and several tests run inside every build. | `test_relationship_gate.py` | Proves a deliberately dangling relationship is rejected. | Useful integration test while local integrity validation exists, not an ordinary-build step. | Move under `tests/`; replace with native-relation fixtures. | | `test-milestone.sh` | Runs repeat builds and a metadata-mutation proof. | Useful milestone evidence, but it causes repeated full builds and is not production machinery. | Retain as a release/acceptance suite, not the deployment entry point. | -The desired outcome is not “no scripts.” A repository-backed, ephemeral -version of a pipeline designed around a live service needs some glue. The -desired outcome is a small generic build adapter with the metadata semantics -left in the schema, qri operations, and templates, and with tests clearly -separated from production transforms. +The desired outcome is not “no scripts.” +A repository-backed, ephemeral version of a pipeline designed around a live service needs some glue. +The desired outcome is a small generic build adapter with the metadata semantics left in the schema, qri operations, and templates, and with tests clearly separated from production transforms. -The current Pages job performs four complete builds: three inside -`test-milestone.sh` and a fourth for deployment. Every one also reruns the two -deliberately broken-record gate proofs. That is strong one-time milestone -evidence, but unnecessary deployment work. The ordinary build should validate -the real snapshot once; mutation, repeat-build, and negative-fixture tests -belong in a separate pull-request or release-verification job. +The current Pages job performs four complete builds: three inside `test-milestone.sh` and a fourth for deployment. +Every one also reruns the two deliberately broken-record gate proofs. +That is strong one-time milestone evidence, but unnecessary deployment work. +The ordinary build should validate the real snapshot once; mutation, repeat-build, and negative-fixture tests belong in a separate pull-request or release-verification job. ### `validate_records.py` in particular -Dump Things exposes `POST //validate/record/` for one -record. Its service CLI starts a service, and `dtc` can import, post, and read -records, but the pinned tools do not expose a command that walks a repository, -validates every source file without storing it, fails once for the collection, -and writes a useful report. The local script supplies that missing batch -operation. +Dump Things exposes `POST //validate/record/` for one record. +Its service CLI starts a service, and `dtc` can import, post, and read records, but the pinned tools do not expose a command that walks a repository, validates every source file without storing it, fails once for the collection, and writes a useful report. +The local script supplies that missing batch operation. -That makes a thin wrapper reasonable. What would be unreasonable is -reimplementing LinkML rules locally; the script does not do that. Its likely -long-term home is an upstream `dtc validate` command or a small reusable build -action, not a CON-specific 101-line script. +That makes a thin wrapper reasonable. +What would be unreasonable is reimplementing LinkML rules locally; the script does not do that. +Its likely long-term home is an upstream `dtc validate` command or a small reusable build action, not a CON-specific 101-line script. -There is also a small snapshot bug in the current wiring: `prepare_build.py` -first copies the metadata into the isolated store, but validation and local -relationship checks then read the original directory while the service and qri -read the copy. All gates should consume the same immutable build snapshot. +There is also a small snapshot bug in the current wiring: `prepare_build.py` first copies the metadata into the isolated store, but validation and local relationship checks then read the original directory while the service and qri read the copy. +All gates should consume the same immutable build snapshot. ### `enrich_projection.py` in particular -This file does much more than its name suggests. It is both a routing table and -a hand-written graph projector: +This file does much more than its name suggests. +It is both a routing table and a hand-written graph projector: - every accepted PID must occur in `INFO`; - labels and depiction paths are duplicated from records or site data; - DOI-safe and CURIE routes are decided per record; - predicates are translated into presentation labels; -- forward relationships and selected backlinks are assembled into `x_*` - fields; and +- forward relationships and selected backlinks are assembled into `x_*` fields; and - roles that were lost during validation are approximated for display. -qri already supports record caching, class/PID selection, direct-record -inlining, reverse-link injection, filtering, and template rendering. It can -even inline the current string targets using `attributes::value`, although it -cannot predicate-filter or reverse-inject those nested compatibility -attributes. The large adapter exists because the records no longer contain -native links that qri understands and because routing was solved with a -five-record table. Fixing those two boundaries should remove almost all of the -file. +qri already supports record caching, class/PID selection, direct-record inlining, reverse-link injection, filtering, and template rendering. +It can even inline the current string targets using `attributes::value`, although it cannot predicate-filter or reverse-inject those nested compatibility attributes. +The large adapter exists because the records no longer contain native links that qri understands and because routing was solved with a five-record table. +Fixing those two boundaries should remove almost all of the file. ## Which hard-coded content is appropriate -Not every `_index.md` file must be generated. A metadata-driven site can still -contain human-authored editorial material. +Not every `_index.md` file must be generated. +A metadata-driven site can still contain human-authored editorial material. Appropriate manual content includes: @@ -226,18 +167,15 @@ Appropriate manual content includes: Content that must be generated from metadata includes: -- the membership of Projects, People, Publications, Outputs, Objectives, and - other entity collections; +- the membership of Projects, People, Publications, Outputs, Objectives, and other entity collections; - entity labels and routes; - roles, authorship, generation, membership, and other relationship lists; - backlinks and related-record groupings; and - graph nodes and edges. -The current section files under `content/{persons,projects,publications, -instruments}/_index.md` each hand-write the only entity link. The homepage and -publication templates also contain milestone-specific narrative. This is why -the implementation feels hard-coded even though individual detail-page titles -and descriptions originate in YAML. +The current section files under `content/{persons,projects,publications, instruments}/_index.md` each hand-write the only entity link. +The homepage and publication templates also contain milestone-specific narrative. +This is why the implementation feels hard-coded even though individual detail-page titles and descriptions originate in YAML. A better separation is: @@ -248,17 +186,15 @@ projection generated class membership, pages, backlinks, and graph data layouts/ presentation of those generated structures ``` -An empty collection should render an intentional empty state. Adding a record -should never require adding a bullet to `_index.md`. +An empty collection should render an intentional empty state. +Adding a record should never require adding a bullet to `_index.md`. ## What was actually taken from `www-from-model` -The candidate correctly avoided merging or grafting the complete upstream -history. At the initial import commit, 20 paths were copied byte-for-byte. By -the current candidate, only six remain exact upstream files—four menu icons, -`markup.toml`, and `taxonomies.toml`—plus the same Congo v2.13.0 submodule pin. -Every imported page template and layout partial was subsequently adapted, and -several are functional replacements rather than small adaptations. +The candidate correctly avoided merging or grafting the complete upstream history. +At the initial import commit, 20 paths were copied byte-for-byte. +By the current candidate, only six remain exact upstream files—four menu icons, `markup.toml`, and `taxonomies.toml`—plus the same Congo v2.13.0 submodule pin. +Every imported page template and layout partial was subsequently adapted, and several are functional replacements rather than small adaptations. | Category | Current result | | --- | --- | @@ -271,38 +207,30 @@ At the reviewed upstream commit, the update workflow: 1. caches all records; 2. selects each class; 3. injects reverse links such as generated outputs and child projects; -4. inlines linked people, roles, objectives, projects, publications, and - identifiers; +4. inlines linked people, roles, objectives, projects, publications, and identifiers; 5. renders entity records as Hugo taxonomy terms; and 6. derives `graph.json` from the same native relationships. -The custom taxonomy and term layouts then turn those terms into class grids, -filters, related-record panels, backlinks, and per-record graphs. The reviewed -upstream content contains 221 taxonomy terms. The candidate produces zero -terms; its entities are ordinary leaf pages and its four class roots contain -manual links. The exact-copy taxonomy configuration additionally exposes empty -Datasets, Objectives, Topics, and Tags roots. - -The graph was not merely a decoration. Upstream's `code/pool2graph.py` turns -Organizations, People, Projects, Publications, Topics, Objectives, Datasets, -and Instruments into nodes and their native relationships into edges. The -homepage and term layout load the renderer, and each term declares a graph root -PID. The current candidate only checks a five-node graph during the build and -renders selected textual links. It publishes neither graph data nor a graph -interface. +The custom taxonomy and term layouts then turn those terms into class grids, filters, related-record panels, backlinks, and per-record graphs. +The reviewed upstream content contains 221 taxonomy terms. +The candidate produces zero terms; its entities are ordinary leaf pages and its four class roots contain manual links. +The exact-copy taxonomy configuration additionally exposes empty Datasets, Objectives, Topics, and Tags roots. + +The graph was not merely a decoration. +Upstream's `code/pool2graph.py` turns Organizations, People, Projects, Publications, Topics, Objectives, Datasets, and Instruments into nodes and their native relationships into edges. +The homepage and term layout load the renderer, and each term declares a graph root PID. +The current candidate only checks a five-node graph during the build and renders selected textual links. +It publishes neither graph data nor a graph interface. ### Why the preview looks completely different -`www-from-model` is not just stock Congo with generated Markdown. Its custom -application layer supplies taxonomy grids, filtering, related-term panels, -backlinks, a hybrid navigation header, and graph views. The candidate removed -that layer, flattened the menu, disabled several theme features, and applied -legacy CON colors, typography, logo treatment, and sparse five-record prose. +`www-from-model` is not just stock Congo with generated Markdown. +Its custom application layer supplies taxonomy grids, filtering, related-term panels, backlinks, a hybrid navigation header, and graph views. +The candidate removed that layer, flattened the menu, disabled several theme features, and applied legacy CON colors, typography, logo treatment, and sparse five-record prose. -Using CON's visual identity was appropriate. Removing the upstream information -architecture was not required by that decision. The better blend is legacy CON -branding and editorial voice applied to the upstream metadata navigation -model. +Using CON's visual identity was appropriate. +Removing the upstream information architecture was not required by that decision. +The better blend is legacy CON branding and editorial voice applied to the upstream metadata navigation model. ## A clearer responsibility model for the three repositories @@ -331,10 +259,7 @@ flowchart LR | `centerforopenneuroscience.org` | Accepted canonical YAML, editorial content, URL policy, CON styling/layout overrides, asset manifest, and deployment configuration. | A copy of migration internals or a growing collection of record-specific compatibility hacks. | | Pinned Orinoco components | Schema validation, service behavior, queries, relationship projection, and graph rendering primitives. | Floating dependencies whose compatibility is inferred from a single successful simple record. | -This model permits selective adoption without losing the essential design: -CON owns the site, while reusable metadata-navigation behavior continues to be -derived from and, where possible, contributed back to `www-from-model` and the -relevant Orinoco component. +This model permits selective adoption without losing the essential design: CON owns the site, while reusable metadata-navigation behavior continues to be derived from and, where possible, contributed back to `www-from-model` and the relevant Orinoco component. ## The native type-discriminator incompatibility @@ -342,86 +267,58 @@ relevant Orinoco component. The reviewed schema represents qualified facts with native containers: -- `Association` connects an agent to a project and carries roles such as - `marcrel:led`; -- `Attribution` connects an author to a publication and carries - `marcrel:aut`; -- `Generation` represents generated outputs or publication venue information; - and +- `Association` connects an agent to a project and carries roles such as `marcrel:led`; +- `Attribution` connects an author to a publication and carries `marcrel:aut`; +- `Generation` represents generated outputs or publication venue information; and - `DOI` and `ISSN` are typed identifier subclasses. -At the candidate's pins—Dump Things service 6.3.6 at `9f101d9`, the vendored -schema from `d26ea41`, and LinkML/LinkML-runtime 1.11.1—validation uses two -generated representations in sequence: +At the candidate's pins—Dump Things service 6.3.6 at `9f101d9`, the vendored schema from `d26ea41`, and LinkML/LinkML-runtime 1.11.1—validation uses two generated representations in sequence: 1. a Pydantic API model validates the JSON/YAML shape; and -2. the internal LinkML loader loads the same data while probing conversion to - RDF. +2. the internal LinkML loader loads the same data while probing conversion to RDF. -For the affected subclasses, the generated Pydantic model accepts a full class -URI or an `xyzri:*` designator. The generated Python loader identifies the -subclass with a `dlschemas:*` CURIE or a URI object and compares it to the -plain string from the Pydantic dump. The older `dlthings:*` aliases present in -the migration input fail earlier in Pydantic. In the tested combination there -is consequently no single YAML string that both generated stages accept for -the affected nested classes. +For the affected subclasses, the generated Pydantic model accepts a full class URI or an `xyzri:*` designator. +The generated Python loader identifies the subclass with a `dlschemas:*` CURIE or a URI object and compares it to the plain string from the Pydantic dump. +The older `dlthings:*` aliases present in the migration input fail earlier in Pydantic. +In the tested combination there is consequently no single YAML string that both generated stages accept for the affected nested classes. -Omitting the DOI or ISSN discriminator can pass by loading the value as the -base `Identifier`, but that is silent semantic loss, not compatibility. +Omitting the DOI or ISSN discriminator can pass by loading the value as the base `Identifier`, but that is silent semantic loss, not compatibility. -This is best understood as a generator/runtime compatibility defect at the -selected pins, plus an ordinary migration from older aliases—not evidence that -qualified relationships are unsuitable for the canonical data. +This is best understood as a generator/runtime compatibility defect at the selected pins, plus an ordinary migration from older aliases—not evidence that qualified relationships are unsuitable for the canonical data. ### What the compatibility matrix found -The simple “choose a different nearby version” solution has now been tested -and ruled out: +The simple “choose a different nearby version” solution has now been tested and ruled out: -- with Dump Things 6.3.6 and schema `d26ea413`, exact LinkML and - LinkML-runtime pairs 1.10.0, 1.11.0, and 1.11.1 all fail all five types in - the same way; -- historical schema commits `e47faa47` (pre-v1 imported schemas) and - `21fd187d` (the first v1 imports/things-v2 switch), each tested with LinkML - 1.10.0 and 1.11.1, fail identically; +- with Dump Things 6.3.6 and schema `d26ea413`, exact LinkML and LinkML-runtime pairs 1.10.0, 1.11.0, and 1.11.1 all fail all five types in the same way; +- historical schema commits `e47faa47` (pre-v1 imported schemas) and `21fd187d` (the first v1 imports/things-v2 switch), each tested with LinkML 1.10.0 and 1.11.1, fail identically; - Dump Things 6.3.7 differs from 6.3.6 only in its changelog; and -- the current LinkML (`e5d97c45`) and LinkML-runtime (`7f98f220`) main branches - still contain the same mismatch. +- the current LinkML (`e5d97c45`) and LinkML-runtime (`7f98f220`) main branches still contain the same mismatch. -The relevant upstream compliance suite already identifies overridden class -URIs as incomplete for Pydantic and Python dataclasses. This is therefore not -a CON-specific schema anomaly. +The relevant upstream compliance suite already identifies overridden class URIs as incomplete for Pydantic and Python dataclasses. +This is therefore not a CON-specific schema anomaly. -A two-part in-memory proof patch succeeded with the current schema and service -pins: +A two-part in-memory proof patch succeeded with the current schema and service pins: -1. include the namespace-compacted explicit class URI—the `dlschemas:*` - value emitted by PythonGenerator—in Pydantic's accepted discriminator set; -2. compare URI-like discriminator values as normalized strings in - `linkml_runtime.utils.yamlutils.YAMLRoot._class_for`. +1. include the namespace-compacted explicit class URI—the `dlschemas:*` value emitted by PythonGenerator—in Pydantic's accepted discriminator set; +2. compare URI-like discriminator values as normalized strings in `linkml_runtime.utils.yamlutils.YAMLRoot._class_for`. -With both changes, native Project and Publication fixtures containing all five -affected types survived JSON -> internal model -> Turtle -> JSON -> Turtle. -The two Turtle graphs were RDF-isomorphic, and lead/author roles and all typed -discriminators remained present. A one-sided normalization is insufficient: -it fixes the first load, but PythonGenerator then emits `dlschemas:*`, which -unpatched Pydantic rejects on the second cycle. +With both changes, native Project and Publication fixtures containing all five affected types survived JSON -> internal model -> Turtle -> JSON -> Turtle. +The two Turtle graphs were RDF-isomorphic, and lead/author roles and all typed discriminators remained present. +A one-sided normalization is insufficient: it fixes the first load, but PythonGenerator then emits `dlschemas:*`, which unpatched Pydantic rejects on the second cycle. ### Why the current workaround is unacceptable as architecture -The workaround replaced the native containers with generic -`AttributeSpecification` objects whose `value` is a PID string. That lets the -five files pass the service, but it has four consequences: +The workaround replaced the native containers with generic `AttributeSpecification` objects whose `value` is a PID string. +That lets the five files pass the service, but it has four consequences: -- lead and author roles are no longer present in machine-readable canonical - records; +- lead and author roles are no longer present in machine-readable canonical records; - qri no longer recognizes those values as native relationship edges; - graph generation no longer sees them; and -- local predicate- and class-specific code must reconstruct navigation and - format validation. +- local predicate- and class-specific code must reconstruct navigation and format validation. -The workaround is useful as a diagnostic probe. It should not be promoted as -the accepted Milestone 1 data model. +The workaround is useful as a diagnostic probe. +It should not be promoted as the accepted Milestone 1 data model. ### Options @@ -435,38 +332,27 @@ the accepted Milestone 1 data model. ### Recommended pin and upgrade contract -Create a small native compatibility fixture containing at least one -`Association`, `Attribution`, `Generation`, `DOI`, and `ISSN`. A candidate pin -is acceptable only when the fixture: +Create a small native compatibility fixture containing at least one `Association`, `Attribution`, `Generation`, `DOI`, and `ISSN`. +A candidate pin is acceptable only when the fixture: 1. validates through the actual Dump Things endpoint; -2. survives JSON -> internal LinkML model -> Turtle -> JSON without losing its - subclass, target, or roles; +2. survives JSON -> internal LinkML model -> Turtle -> JSON without losing its subclass, target, or roles; 3. survives `dtc` export and qri caching; 4. can be discovered by qri direct inlining and reverse-link injection; and 5. produces the expected taxonomy/backlink and graph edges. The bounded version matrix is complete and found no existing compatible tuple. -The next step is to turn the successful two-part proof into upstream tests and -focused fixes in: - -- `linkml/generators/common/type_designators.py`, so the Pydantic accepted set - includes PythonGenerator's namespace-compacted explicit class URI; and -- `linkml_runtime/utils/yamlutils.py` in LinkML-runtime, so - `YAMLRoot._class_for` dispatches URI objects and equivalent strings to the - same subclass. - -The upstream compliance test should cover an imported class with an overridden -URI and require agreement between the Pydantic and Python generators. Dump -Things should add a regression fixture containing all five native types through -`FormatConverter` and the validation endpoint. Until releases contain those -fixes, pin the exact tested LinkML and LinkML-runtime fork commits alongside -the existing exact schema and service pins. - -After the fix, migrate the old `dlthings:*` type aliases in migration input to -the canonical `dlschemas:things-prov/v1/*` and -`dlschemas:things-publications/v1/*` discriminators emitted by the generated -model. That is a reviewable schema migration, not a site projection hack. +The next step is to turn the successful two-part proof into upstream tests and focused fixes in: + +- `linkml/generators/common/type_designators.py`, so the Pydantic accepted set includes PythonGenerator's namespace-compacted explicit class URI; and +- `linkml_runtime/utils/yamlutils.py` in LinkML-runtime, so `YAMLRoot._class_for` dispatches URI objects and equivalent strings to the same subclass. + +The upstream compliance test should cover an imported class with an overridden URI and require agreement between the Pydantic and Python generators. +Dump Things should add a regression fixture containing all five native types through `FormatConverter` and the validation endpoint. +Until releases contain those fixes, pin the exact tested LinkML and LinkML-runtime fork commits alongside the existing exact schema and service pins. + +After the fix, migrate the old `dlthings:*` type aliases in migration input to the canonical `dlschemas:things-prov/v1/*` and `dlschemas:things-publications/v1/*` discriminators emitted by the generated model. +That is a reviewable schema migration, not a site projection hack. Upgrades should then be deliberate migrations: @@ -479,60 +365,44 @@ change one or more candidate pins -> update the exact lock and provenance together ``` -That creates schema-driven updates rather than a growing list of downstream -predicate hacks. +That creates schema-driven updates rather than a growing list of downstream predicate hacks. ## Assets and git-annex -The previous progress report's conclusion about Yaroslav's photograph was -incorrect. The legacy path is an annex pointer with key -`MD5E-s37940--90e74fa17a709006dd527c5b36e41217.jpg`. `git annex whereis` -currently lists only the local repository, but -`git annex checkpresentkey datasets.datalad.org` succeeds. The payload is -available from the configured `datasets.datalad.org` remote. `whereis` reads -the locally available git-annex location log; it does not probe each content -remote. The GitHub/fork annex branch has stale location metadata, while the -remote itself and its annex branch report the key. Falling back to the deployed -website before testing the remote directly was premature. - -The candidate copied the photograph into `assets/img` as an ordinary Git blob -and added an `annex.largefiles=nothing` exception. That should be reversed. The -final Hugo asset path can itself be annexed, with the required key fetched -before Hugo runs. - -The CON raster and SVG logos and the selected DataLad logo are different: at -the legacy commit they were already ordinary Git blobs, not annex pointers. -Their `assets/img` copies have the same Git object IDs as the legacy paths, so -Git does not store a second payload. Those paths were added because the -Hugo/Congo asset pipeline expects processable assets under `assets/`. We should -nevertheless choose an explicit asset policy rather than infer one file at a -time. +The previous progress report's conclusion about Yaroslav's photograph was incorrect. +The legacy path is an annex pointer with key `MD5E-s37940--90e74fa17a709006dd527c5b36e41217.jpg`. +`git annex whereis` currently lists only the local repository, but `git annex checkpresentkey datasets.datalad.org` succeeds. +The payload is available from the configured `datasets.datalad.org` remote. +`whereis` reads the locally available git-annex location log; it does not probe each content remote. +The GitHub/fork annex branch has stale location metadata, while the remote itself and its annex branch report the key. +Falling back to the deployed website before testing the remote directly was premature. + +The candidate copied the photograph into `assets/img` as an ordinary Git blob and added an `annex.largefiles=nothing` exception. +That should be reversed. +The final Hugo asset path can itself be annexed, with the required key fetched before Hugo runs. + +The CON raster and SVG logos and the selected DataLad logo are different: at the legacy commit they were already ordinary Git blobs, not annex pointers. +Their `assets/img` copies have the same Git object IDs as the legacy paths, so Git does not store a second payload. +Those paths were added because the Hugo/Congo asset pipeline expects processable assets under `assets/`. +We should nevertheless choose an explicit asset policy rather than infer one file at a time. Recommended policy: -- keep small existing branding files in ordinary Git unless CON chooses an - all-binary annex policy; -- keep photographs, large media, graph bundles, and generated depictions in - git-annex; +- keep small existing branding files in ordinary Git unless CON chooses an all-binary annex policy; +- keep photographs, large media, graph bundles, and generated depictions in git-annex; - maintain a manifest of build-required asset paths and keys; - fetch only that manifest from an availability-tested remote before Hugo; - fail early if any required key is unavailable; and -- repair stale annex location metadata rather than treating `whereis` alone as - an availability test. - -A fresh Pages checkout should fetch the `git-annex` branch before -initialization, enable the DataLad remote over read-only HTTPS, repair or verify -the location log with a remote-scoped fast `fsck`, and then `get` only the -manifest paths. The built Pages artifact contains ordinary dereferenced media; -site visitors do not need git-annex. Any repaired annex metadata should be -pushed deliberately, separately from the site-source change. - -For GitHub Actions on Linux, both Hugo and git-annex can be pinned from -conda-forge through Pixi. Hugo 0.154.5 Extended is also available there for -macOS arm64 and Windows. git-annex is currently packaged there for Linux -x86-64, but not for macOS, Windows, or Linux ARM. Local Mac development still -needs a system installation such as the existing Homebrew package. This does -not prevent Pixi from owning the site build and the Linux x86-64 CI toolchain. +- repair stale annex location metadata rather than treating `whereis` alone as an availability test. + +A fresh Pages checkout should fetch the `git-annex` branch before initialization, enable the DataLad remote over read-only HTTPS, repair or verify the location log with a remote-scoped fast `fsck`, and then `get` only the manifest paths. +The built Pages artifact contains ordinary dereferenced media; site visitors do not need git-annex. +Any repaired annex metadata should be pushed deliberately, separately from the site-source change. + +For GitHub Actions on Linux, both Hugo and git-annex can be pinned from conda-forge through Pixi. +Hugo 0.154.5 Extended is also available there for macOS arm64 and Windows. git-annex is currently packaged there for Linux x86-64, but not for macOS, Windows, or Linux ARM. +Local Mac development still needs a system installation such as the existing Homebrew package. +This does not prevent Pixi from owning the site build and the Linux x86-64 CI toolchain. ## Recommended metadata-driven site baseline @@ -552,75 +422,52 @@ flowchart TD H --> P ``` -The first redesign should restore only the subset required by the reviewed CON -slice, but it should be generic across records: +The first redesign should restore only the subset required by the reviewed CON slice, but it should be generic across records: -- Organization, Person, Project, Publication, and Instrument are generated - members of their classes; +- Organization, Person, Project, Publication, and Instrument are generated members of their classes; - their native roles and links drive forward navigation and backlinks; - the same record stream generates the graph; - an added valid record automatically appears in its class index; -- routes follow a generic PID convention with an optional explicit site-path - annotation for exceptions such as DOI URLs; -- depictions are associated through metadata or a generic bundle convention, - not a Python PID table; and -- editorial `_index.md` content augments generated lists rather than encoding - them. - -The pinned upstream `term.html` is a useful starting point for the requested -graph placement: it already puts a depiction in a 30% table cell and the graph -beside it. For CON, the detail-page graph should start smaller—approximately a -30–35% side panel with a 280–320 pixel height cap on desktop—and stack or -collapse below the main record content on narrow screens. A separate Explore -page can retain a larger graph. Exact dimensions and controls belong in the -focused visual-design discussion, but the graph should be part of the data -contract, not an optional late decoration. +- routes follow a generic PID convention with an optional explicit site-path annotation for exceptions such as DOI URLs; +- depictions are associated through metadata or a generic bundle convention, not a Python PID table; and +- editorial `_index.md` content augments generated lists rather than encoding them. + +The pinned upstream `term.html` is a useful starting point for the requested graph placement: it already puts a depiction in a 30% table cell and the graph beside it. +For CON, the detail-page graph should start smaller—approximately a 30–35% side panel with a 280–320 pixel height cap on desktop—and stack or collapse below the main record content on narrow screens. +A separate Explore page can retain a larger graph. +Exact dimensions and controls belong in the focused visual-design discussion, but the graph should be part of the data contract, not an optional late decoration. ## Proposed revised Milestone 1 acceptance criteria Keep the existing deterministic-build and static-deployment criteria, and add: -1. canonical records use native `Association`, `Attribution`, `Generation`, - `DOI`, and `ISSN` where the source calls for them; +1. canonical records use native `Association`, `Attribution`, `Generation`, `DOI`, and `ISSN` where the source calls for them; 2. role and relationship semantics survive the full validation and qri path; -3. adding a fixture record, without code or index edits, creates its page and - class-list membership; +3. adding a fixture record, without code or index edits, creates its page and class-list membership; 4. forward links and at least one derived backlink are rendered from metadata; -5. graph nodes and edges are generated from the same canonical records and a - compact graph appears on entity pages; +5. graph nodes and edges are generated from the same canonical records and a compact graph appears on entity pages; 6. no record PID inventory exists in projection code; 7. required annexed assets are fetched from a verified remote in CI; and -8. the ordinary build runs once, while negative and repeat-build proofs live in - a separate acceptance suite. +8. the ordinary build runs once, while negative and repeat-build proofs live in a separate acceptance suite. ## Focused decisions before implementation resumes -1. Confirm that Milestone 1 should be reopened around the stronger acceptance - criteria above rather than treating PR 84 as the migration baseline. -2. Decide whether to retain Hugo taxonomies exactly as upstream or implement - equivalent generated class sections. Reusing the upstream taxonomy/term - machinery is the lower-risk starting point. -3. Agree on the route contract: generic PID-derived paths plus explicit - metadata/site overrides for exceptions is recommended. -4. Confirm graph scope, initial node/edge classes, compact detail placement, - and larger Explore behavior. -5. Confirm the asset policy. Preserving ordinary-Git logos and annexing - photographs/large media matches the legacy repository's actual history. -6. Adopt Pixi as the single build entry point and exact Hugo/Python lock, with - system git-annex allowed on macOS and Pixi-pinned git-annex in Linux CI. +1. Confirm that Milestone 1 should be reopened around the stronger acceptance criteria above rather than treating PR 84 as the migration baseline. +2. Decide whether to retain Hugo taxonomies exactly as upstream or implement equivalent generated class sections. +Reusing the upstream taxonomy/term machinery is the lower-risk starting point. +3. Agree on the route contract: generic PID-derived paths plus explicit metadata/site overrides for exceptions is recommended. +4. Confirm graph scope, initial node/edge classes, compact detail placement, and larger Explore behavior. +5. Confirm the asset policy. +Preserving ordinary-Git logos and annexing photographs/large media matches the legacy repository's actual history. +6. Adopt Pixi as the single build entry point and exact Hugo/Python lock, with system git-annex allowed on macOS and Pixi-pinned git-annex in Linux CI. ## Bounded next step -Do not begin the full CON migration. First run three small spikes against the -same five records: +Do not begin the full CON migration. +First run three small spikes against the same five records: -1. land or temporarily pin the tested two-part LinkML discriminator fix and - restore the native relationship/identifier fixtures; -2. restore metadata-derived class lists, backlinks, and graph generation from - the reviewed upstream code; and -3. restore the Yaroslav image as an annexed build input and replace the Hugo - installer with Pixi. +1. land or temporarily pin the tested two-part LinkML discriminator fix and restore the native relationship/identifier fixtures; +2. restore metadata-derived class lists, backlinks, and graph generation from the reviewed upstream code; and +3. restore the Yaroslav image as an annexed build input and replace the Hugo installer with Pixi. -Once those pass the revised acceptance criteria and the compact graph layout is -reviewed, update the execution plan and decide whether PR 84 should be revised -in place or superseded by a cleaner branch. +Once those pass the revised acceptance criteria and the compact graph layout is reviewed, update the execution plan and decide whether PR 84 should be revised in place or superseded by a cleaner branch. diff --git a/docs/milestone-1-progress.md b/docs/milestone-1-progress.md index bf677dc..5d5c41f 100644 --- a/docs/milestone-1-progress.md +++ b/docs/milestone-1-progress.md @@ -4,37 +4,28 @@ Status: complete — revised metadata-navigation prototype Completed: 2026-07-31 -The revised CON vertical slice is implemented at -`2621231d27b70fb425107a132159f7a9e0d99cda` on `orinoco-lite`. +The revised CON vertical slice is implemented at `2621231d27b70fb425107a132159f7a9e0d99cda` on `orinoco-lite`. - Preview: - Successful Pages run: - Candidate fork branch: -- Original review PR 84: - (closed by its author; it was not reopened by this revision) +- Original review PR 84: (closed by its author; it was not reopened by this revision) ## What changed after the design review -The first candidate proved that an ephemeral Dump Things service, qri, Hugo, -and GitHub Pages could work together, but it did not preserve enough of -`www-from-model`'s metadata-navigation layer. The revised prototype restores -that layer selectively rather than merging upstream history: +The first candidate proved that an ephemeral Dump Things service, qri, Hugo, and GitHub Pages could work together, but it did not preserve enough of `www-from-model`'s metadata-navigation layer. +The revised prototype restores that layer selectively rather than merging upstream history: - Hugo taxonomy and term layouts generate class lists and record pages; -- metadata relationships generate forward terms and Hugo-derived reverse - backlinks; +- metadata relationships generate forward terms and Hugo-derived reverse backlinks; - collection pages retain the upstream filter/list behavior; - qri cache, list, inline, and render stages remain in the build; -- the pinned Things graph renderer produces a compact side graph from the same - nodes and edges; and -- CON branding, editorial introductions, compatibility routes, and deployment - policy remain downstream concerns. +- the pinned Things graph renderer produces a compact side graph from the same nodes and edges; and +- CON branding, editorial introductions, compatibility routes, and deployment policy remain downstream concerns. -Checked-in collection `_index.md` files now contain only editorial text and -display settings. Entity membership, labels, paths, links, backlinks, and -graph data are generated from the canonical YAML. The build rejects nested -hand-authored entity Markdown and the publication checker rejects any extra -entity route, so committed content cannot silently override metadata. +Checked-in collection `_index.md` files now contain only editorial text and display settings. +Entity membership, labels, paths, links, backlinks, and graph data are generated from the canonical YAML. +The build rejects nested hand-authored entity Markdown and the publication checker rejects any extra entity route, so committed content cannot silently override metadata. ## Exit criteria @@ -53,75 +44,47 @@ entity route, so committed content cannot silently override metadata. ## Acceptance evidence -The locked local contract completed successfully after the final source -changes: +The locked local contract completed successfully after the final source changes: -- baseline: 5 metadata pages, 6 graph edges, 21 Hugo pages, 52 files, and 594 - checked internal links; +- baseline: 5 metadata pages, 6 graph edges, 21 Hugo pages, 52 files, and 594 checked internal links; - identical repeat build with the same manifest; -- extension fixture: 6 metadata pages, 7 graph edges, 22 Hugo pages, 53 files, - and 630 checked internal links; +- extension fixture: 6 metadata pages, 7 graph edges, 22 Hugo pages, 53 files, and 630 checked internal links; - upstream schema-invalid fixture rejected; - dangling URL/DOI target rejected by the generic relationship boundary; - duplicate route and hand-authored entity-route fixtures rejected; and - the pinned LinkML discriminator reproducer fails in the documented way. -The local browser check also confirmed that the graph initializes without -console errors, occupies a small 4:3 side panel on desktop, exposes an -accessible relationship-link alternative, and follows record links. The -publication list filters by Yaroslav's generated author metadata, and the -DataLad page shows its forward person relationship and reverse instrument, -organization, and publication backlinks. +The local browser check also confirmed that the graph initializes without console errors, occupies a small 4:3 side panel on desktop, exposes an accessible relationship-link alternative, and follows record links. +The publication list filters by Yaroslav's generated author metadata, and the DataLad page shows its forward person relationship and reverse instrument, organization, and publication backlinks. ## Build and source boundaries The ordinary publication build now has six scripts with distinct roles: -1. `build.sh` orchestrates the ephemeral service, direct `dtc` post, qri, - renderer, Hugo, and final site check. +1. `build.sh` orchestrates the ephemeral service, direct `dtc` post, qri, renderer, Hugo, and final site check. 2. `prepare_build.py` creates the immutable snapshot and disposable workspace. -3. `project_records.py` supplies the one generic temporary schema boundary and - emits routes, relations, taxonomies, and graph data. -4. `check_site.py` verifies rendered output rather than revalidating source - metadata. -5. `test-milestone.sh` owns repeat, extension, and negative acceptance tests; - it is not rerun inside the Pages deployment build. +3. `project_records.py` supplies the one generic temporary schema boundary and emits routes, relations, taxonomies, and graph data. +4. `check_site.py` verifies rendered output rather than revalidating source metadata. +5. `test-milestone.sh` owns repeat, extension, and negative acceptance tests; it is not rerun inside the Pages deployment build. 6. `reproduce_schema_discriminator.py` isolates the upstream LinkML defect. -The former record validator, fixed five-record enrichment table, individual -gate scripts, Hugo downloader, uv project, and uv lock were removed. Pixi -0.73.0 installs the checked-in cross-platform lock; Linux CI also receives the -pinned git-annex package. +The former record validator, fixed five-record enrichment table, individual gate scripts, Hugo downloader, uv project, and uv lock were removed. +Pixi 0.73.0 installs the checked-in cross-platform lock; Linux CI also receives the pinned git-annex package. -The clean `www-from-model` mirror remains at -`6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce`. The CON branch selectively adapts -the taxonomy, term, relationship, filter, qri, and graph patterns; it does not -merge or graft upstream history. `dump-research-info` remains migration -evidence only and is not a build dependency. +The clean `www-from-model` mirror remains at `6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce`. +The CON branch selectively adapts the taxonomy, term, relationship, filter, qri, and graph patterns; it does not merge or graft upstream history. +`dump-research-info` remains migration evidence only and is not a build dependency. ## Accepted schema exception -The pinned Pydantic and Python LinkML model families do not round-trip a common -type discriminator for native `Association`, `Attribution`, `Generation`, -`DOI`, and `ISSN` subclasses. The exact failure, minimal reproducer, attempted -solutions, trade-offs, tested pin-update strategy, and removal condition are -documented in the site repository at -`docs/upstream-schema-discriminator-issue.md` and in -`provenance/schema-compatibility.yaml`. - -Milestone 1 therefore retains one bounded generic projection. It contains no -CON PID, label, route, image, or per-record table; checks every configured -relationship target independent of PID syntax; and keeps the original -source/predicate/target assertions on projected edges. Native qualified roles -and typed identifiers remain explicitly deferred until an upstream-compatible -schema/service/client/qri tuple passes the recorded fixtures. Updating a pin -should then regenerate the model and remove normalization, not add another -version-specific hack. +The pinned Pydantic and Python LinkML model families do not round-trip a common type discriminator for native `Association`, `Attribution`, `Generation`, `DOI`, and `ISSN` subclasses. +The exact failure, minimal reproducer, attempted solutions, trade-offs, tested pin-update strategy, and removal condition are documented in the site repository at `docs/upstream-schema-discriminator-issue.md` and in `provenance/schema-compatibility.yaml`. + +Milestone 1 therefore retains one bounded generic projection. +It contains no CON PID, label, route, image, or per-record table; checks every configured relationship target independent of PID syntax; and keeps the original source/predicate/target assertions on projected edges. +Native qualified roles and typed identifiers remain explicitly deferred until an upstream-compatible schema/service/client/qri tuple passes the recorded fixtures. +Updating a pin should then regenerate the model and remove normalization, not add another version-specific hack. ## Deferred scope -The full CON metadata migration, native qualified roles after the upstream -fix, production cutover and DNS, detailed visual/theming work, broader -information organization, reusable action/template extraction, graphical -editing, published RDF/JSONL contracts, and secondary projections remain later -milestones. +The full CON metadata migration, native qualified roles after the upstream fix, production cutover and DNS, detailed visual/theming work, broader information organization, reusable action/template extraction, graphical editing, published RDF/JSONL contracts, and secondary projections remain later milestones. diff --git a/docs/milestone-2-acceptance.md b/docs/milestone-2-acceptance.md new file mode 100644 index 0000000..fb178b7 --- /dev/null +++ b/docs/milestone-2-acceptance.md @@ -0,0 +1,57 @@ +# Milestone 2 acceptance + +Status: accepted local checkpoint on 2026-08-11 + +Milestone 2 restored the reviewed legacy-equivalent CON public experience on top of the generalized upstream-compatible profile. +The user accepted this milestone as the baseline for broader publication ingestion and deployment work. + +## Accepted commits + +- parent `codex/full-con-migration`: `7ce44a28c13954e514c8b7e9ab6f1eaade77d891`; +- site `codex/full-con-migration`: `d60f274b4bf8af3e513d83d1727cfe3e6c9bb8af`; +- reviewed upstream website base: `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8`; and +- accepted clean-migration checkpoints remain the commits recorded in `docs/full-con-migration.md`. + +These refs are historical checkpoints. +Milestone 3 descends from them on new branches and must not rewrite them. + +## Accepted coverage + +The accepted site contains: + +- 60 canonical records and 8 reference records; +- all 33 reviewed people in the four legacy presentation groups; +- all 23 reviewed featured projects in the six legacy categories; +- 60 graph nodes and 93 native relationship edges; +- 59 generated metadata pages; +- the homepage, About, People, Projects, Engage, Support, Contact, and Explore experience; and +- 71 declared assets with explicit ordinary-Git, annex, unavailable, and absent-in-source handling. + +The organization remains graph-only because the upstream profile has no organization detail route. +The distinguished `xyzrins:.` project remains the canonical homepage root. + +## Acceptance evidence + +The final local acceptance run passed: + +- 82 unit and contract tests; +- 7 Playwright scenarios across Chromium and WebKit; +- two byte-identical metadata projection renders; +- two byte-identical 335-file root static builds; +- the project-path static build and link audit; +- exact graph, route, collection, editor-boundary, and German-isolation checks; and +- cleanup checks for services, probes, tokens, temporary hydration remotes, and local-clone state. + +## Carried review items + +The following are reviewed differences rather than Milestone 2 acceptance failures: + +- EMBER's Brock Wester association and OpenNeuro's Russell Poldrack association remain deferred until supporting-person visibility is decided; +- Chris Markiewicz's exact legacy portrait is unavailable; +- four projects have no source artwork and use the declared neutral fallback; +- the legacy Twitter link remains omitted pending ownership review; +- sixteen selected assets depend on declared read-only annex remotes; and +- publication breadth, static hosting, and downloadable review changes move to Milestone 3. + +Human review may correct any of these matters through ordinary Milestone 3 content commits. +It does not reopen or rewrite the accepted checkpoint. diff --git a/docs/milestone-3-acceptance.md b/docs/milestone-3-acceptance.md new file mode 100644 index 0000000..9fb53c2 --- /dev/null +++ b/docs/milestone-3-acceptance.md @@ -0,0 +1,132 @@ +# Milestone 3 implementation acceptance + +Status: implementation complete; draft human review open + +This report accepts the Milestone 3 implementation as the review candidate. +It does not accept the site's content on behalf of its maintainers, merge the draft pull request, or authorize a production-domain cutover. + +## Published review surfaces + +- Preview: +- Static editor: +- Draft parent pull request: +- Representative person: +- Representative project: +- Representative publication: + +No component pull request was opened. +The component branches exist only so a public recursive checkout can resolve the exact parent gitlinks. + +## Accepted implementation pins + +| Component | Branch | Accepted commit | +| --- | --- | --- | +| Parent coordinator | `codex/milestone-3` | recorded by the draft PR head | +| CON site | `codex/milestone-3` | `26907c487efaa2c31bba9d02398aa201ab6f774b` | +| Zotero ingestion | `codex/milestone-3-zotero` | `062da59cb5a00ca128b3df895426a54088bfc625` | +| Pool UI wrapper | `codex/milestone-3-dependency-refresh` | `93961ace8d4ceaea088ccc04526a9bc5428139a6` | +| SHACL Vue | `codex/milestone-3-dependency-refresh` | `3be33196f0eb7a65817df78b88ea40ecbb5eca11` | + +The immutable clean-migration checkpoints remain named and exact: + +- parent `codex/clean-migration` at `f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d`; and +- site `codex/clean-migration` at `a122e506de9e4a13473edbe8d74a950d74032a16`. + +The workflow fetches only those named checkpoint refs and rejects a missing or moved checkpoint before building the successor. + +## Publication migration result + +The reviewed source is public Zotero group `6197458`, library version 451. +The capture contains five collections and 197 top-level items. +The deterministic selection uses 134 items, excludes 55 items in `External`, and leaves eight unfiled items out of the public promotion. + +The ingestion yields 126 publications, 20 publication-venue candidates, four datasets, and three instruments. +The site promotes all 126 publications, one reviewed Neuroimaging topic, and the required bibliographic reference closure. +Venue candidates remain ingestion evidence because the source's generic publishing placeholder is not a canonical native activity. + +The final projection contains: + +- 186 canonical and 13 reference records; +- 186 graph nodes and 467 native edges; and +- 185 rendered record pages. + +The evidence preserves six DOI duplicate groups, 1,817 unresolved creator observations covering 1,221 names, 42 venue observations, and 49 topic observations covering 36 tags. +These are review queues, not silently invented records. + +## Static preview and editor result + +The Pages deployment is backend-free. +It contains no Dump Things service, browser credential, service token, GitHub token, `CNAME`, or production-domain redirect. +Generated edit links open the static SHACL Vue bundle under the Pages project path. + +SHACL Vue loads digest-bound public records and shapes, saves changes only to an in-memory review queue, and downloads an RDF review bundle. +The checked-in local helper verifies the site commit, source digest, PID, class, path, schema, native relationship closure, and clean checkout before showing or applying canonical YAML changes. +It does not create a branch or pull request. + +Playwright covers the project-path navigation, record load, edit, save, download, and local dry-run validation. +It also proves that the browser sends no write request and ignores or removes service-token state in static mode. + +## Dependency result + +The dependency refresh reduces 23 original package findings to zero production findings. +Four findings remain in the development-only stable VitePress documentation chain: three moderate and one high. +There is no supported stable upgrade or automatic fix; forcing VitePress 2 alpha or an out-of-range Vite is not accepted. +Question M3-Q017 records the human exception. + +The production build, lockfile, unit tests, documentation build, Markdown sanitization regression, external-URL guard, and parent browser tests pass. +The wrapper independently builds the editor twice and rejects byte differences. + +## Reproducibility and hosted evidence + +The final gate uses a public recursive clone with an empty home directory, no system or global Git configuration, no credentials, no ambient Git identity, and no interactive authentication. +It verifies every one of the 28 recursive gitlinks and all public component origins, then builds and exercises the Pages artifact. + +Two editor builds and two complete Pages builds are byte-identical. +The branch checkout and detached recursive checkout produce the same editor and Pages trees. +Profile-local presentation overrides use the reviewed source images directly rather than invoking Hugo's platform-dependent responsive-image encoder, so Linux and macOS do not generate competing derived pixels. +The Engage page loads lightweight poster previews and leaves the preserved print-resolution artwork behind explicit links. +Quicklink prefetching is disabled for the CON profile and rejected by the artifact audit, so those large originals remain click-driven. + +The final local gate produced two identical 144-file editor bundles and two identical 444-file Pages artifacts. +The backend-free site subset has digest `2c88cd971b5ba90c47d1a6a49bfc45636baac7336c3de1116f7de3128f5b3442`; the complete pre-publication payload has digest `18abe466822a8a2d95d47f2a4b60eeac81e2e9630e4aaa05e2a9da73b17c631e`. +Hugo processed zero images, while the 186-node graph and 185 edit routes remain unchanged. + +The uploaded archive includes the audited root dotfiles so its file counts and hashes match `publication.json`. +Pull-request runs build and upload an artifact but cannot enter the deploy job. +Pushes to the temporary review branch deploy through the protected `github-pages` environment. + +## Bounded technical debt + +The following issues are intentionally visible for the broader human review: + +- the editor still requests the Roboto font from Google, so the preview is backend-free but not fully offline or independently archived; +- four development-only VitePress audit findings await a supported stable dependency line; +- sixteen annex-backed assets depend on exact read-only remote objects, and the unavailable Chris Markiewicz portrait uses the declared neutral fallback; +- creator, venue, topic, hidden-person, and duplicate-publication reconciliation remains bounded by the source evidence described above; +- applying a multi-record editor bundle is safe per file but not a transactional all-or-nothing filesystem operation; and +- the single Pages environment is a shared branch preview, not a distinct URL for every pull request. + +These items do not require a persistent service and do not block content review. +They must not be misrepresented as resolved production policy. + +## Human decisions still required + +The complete decision register is [`milestone-3-decisions.md`](milestone-3-decisions.md). +In particular, maintainers must decide: + +- which unresolved creators, venues, tags, duplicates, and supporting people to promote; +- whether the static editor's patches should target the CON repository or the account mirror, and whether the two-repository content/gitlink boundary stays; +- whether canonical YAML may remain available as the editor's bulk public catalog; +- whether named Pages deployment reviewers are required; +- whether to remove the temporary milestone-branch deployment after merge; +- whether the external font and development-only advisory exceptions are acceptable; and +- whether, after separate content approval, this preview should move toward the production custom domain. + +The safe defaults remain fail-closed. +None of these questions authorizes hosted authentication, automatic pull-request submission, Zotero writes, DNS changes, or production cutover. + +## Completion statement + +Milestone 3 is implementation-complete and ready for comprehensive human review. +The draft pull request remains deliberately unmerged and the preview remains a review deployment. +Human content acceptance and production policy are the next phase, not implicit consequences of this technical acceptance. diff --git a/docs/milestone-3-decisions.md b/docs/milestone-3-decisions.md new file mode 100644 index 0000000..222f8de --- /dev/null +++ b/docs/milestone-3-decisions.md @@ -0,0 +1,56 @@ +# Milestone 3 decision register + +Status: implementation complete; draft human review open + +This register distinguishes decisions already authorized by the user from questions that require human review. +Implementations must preserve unresolved source evidence and fail closed rather than invent an answer. + +## Accepted decisions + +| ID | Decision | Consequence | +| --- | --- | --- | +| M3-D001 | Milestone 2 is accepted at the commits recorded in `milestone-2-acceptance.md`. | Milestone 3 uses new successor branches and does not rewrite the accepted parent or site refs. | +| M3-D002 | Broader publication coverage comes from repeatable public Zotero API ingestion, not from copying the old snapshot alone. | The source snapshot records pagination, item and library versions, collection membership, and capture metadata before transformation. | +| M3-D003 | The only pull request in this milestone targets `con/orinoco-lite-dev:main`. | Required submodule commits may be pushed for gitlink availability, but no submodule PR is opened. | +| M3-D004 | The public preview is the parent repository's GitHub Pages project site. | The canonical preview base path is `/orinoco-lite-dev/`; no custom domain or production redirect is configured. | +| M3-D005 | Browser editing remains credential-free. | SHACL Vue loads public static data and downloads a review bundle; a local authenticated checkout prepares any later PR. | +| M3-D006 | Hosted authentication and direct pull-request creation remain deferred. | No OAuth, GitHub App, browser token, or persistent metadata write service is introduced. | +| M3-D007 | Production dependency advisories are resolved by updating the pinned SHACL Vue stack. | Exact dependency and lockfile changes must pass upstream and parent browser regression tests; unsupported overrides are not used, and any development-only exception is recorded separately. | + +## Existing source-policy decisions retained + +The Zotero importer retains the reviewed source decisions already recorded in `submodules/dump-research-info`: + +- Zotero remains authoritative for the CON publication feed; +- ingestion and curation remain separate; +- creator mappings are exact and reviewed, never fuzzy at promotion time; +- all named collections except the reviewed `External` exclusion are eligible, while unfiled or unsupported items go to review; and +- Zotero writes require a separate reviewed-additions record and are not part of this milestone's ingestion path. + +## Human review required + +| ID | Question | Safe implementation default | +| --- | --- | --- | +| M3-Q001 | Which unresolved Zotero creator identities should become new public people records rather than remain literal source creators? | Do not create people automatically; retain the publication candidate and unresolved creator evidence. | +| M3-Q002 | Which DOI duplicates represent the same publication, alternate versions, corrections, or distinct outputs? | Merge only the transformer's reviewed exact duplicate class; report every ambiguous group. | +| M3-Q003 | Should missing or ambiguous publication venues be modeled from authoritative registry enrichment? | Retain the Zotero venue literal and queue a venue decision; do not invent ISSNs or venue identity. | +| M3-Q004 | Should items in `External`, unfiled, or unsupported Zotero categories appear publicly? | Follow the existing eligibility rule and report them without publishing them. | +| M3-Q005 | Are the two deferred project-person associations now ready for public supporting-person records? | Keep the Milestone 2 deferrals until a content owner approves visibility. | +| M3-Q006 | Is the unavailable Chris Markiewicz portrait replaceable with a newly reviewed image and license? | Keep the neutral declared fallback. | +| M3-Q007 | Is the legacy CON social-media account still owned and appropriate to publish? | Keep the Twitter link omitted. | +| M3-Q008 | Is continued read-only remote custody sufficient for the sixteen annex-backed assets? | Hydrate by exact key and digest for the preview; do not claim durable custody. | +| M3-Q009 | After review, should the preview replace the production custom-domain site? | Do not change DNS, redirects, `CNAME`, or production settings in Milestone 3. | +| M3-Q010 | Who may approve canonical publication and editorial changes after the draft PR is opened? | Require ordinary human PR review; do not encode an unapproved CODEOWNERS or branch-protection policy. | +| M3-Q011 | Which public site repository should receive a patch downloaded by the static editor? | Generate a repository-relative patch at the pinned site commit, but do not claim a contribution target until the CON repository or account mirror is selected. | +| M3-Q012 | Should Pages continue to deploy from `codex/milestone-3` after review? | Use that branch only to bootstrap the pre-merge preview; then remove its push trigger and deploy only from `main`. | +| M3-Q013 | Should canonical content retain the site-repository plus parent-gitlink review boundary? | Preserve the proven two-repository ownership in this milestone; consider moving content only as a separately reviewed architecture change. | +| M3-Q014 | May the Pages editor publish a bulk catalog containing the canonical YAML verbatim? | Permit it because the YAML is already public website input, but call out the increased convenience of bulk download during human review. | +| M3-Q015 | Should the `github-pages` environment require named deployment reviewers? | Use ordinary repository controls without inventing a reviewer policy; add environment protection only after maintainers choose the approvers. | +| M3-Q016 | May deployment configuration or schema-authored markup become writable by untrusted users in a later hosted editor? | Treat those inputs as trusted, pinned build inputs in this static preview; require an explicit sanitization boundary before making them user-controlled. | +| M3-Q017 | Is the stable documentation toolchain's four-item development-only advisory exception acceptable until VitePress publishes a supported patched line? | Keep documentation out of the deployed runtime, require a zero-finding production audit, and do not force an unsupported Vite or VitePress alpha override. | +| M3-Q018 | May the preview editor continue loading the Roboto font from Google? | Allow it in this review preview, but vendor or remove the font before claiming an offline or independently archived deployment. | + +## Completion updates + +The exact ingestion totals, exclusions, dependency-audit results, public review links, workflow evidence, and remaining debt are recorded in [`milestone-3-acceptance.md`](milestone-3-acceptance.md). +Questions M3-Q001 through M3-Q018 remain open for human review; none is silently resolved by the preview implementation. diff --git a/docs/milestone-3-dependencies.md b/docs/milestone-3-dependencies.md new file mode 100644 index 0000000..2133984 --- /dev/null +++ b/docs/milestone-3-dependencies.md @@ -0,0 +1,51 @@ +# Milestone 3 SHACL Vue dependency review + +Status: implementation complete; bounded exception awaiting human review + +Nested SHACL Vue commit: `3be33196f0eb7a65817df78b88ea40ecbb5eca11` + +Pool UI wrapper commit: `93961ace8d4ceaea088ccc04526a9bc5428139a6` + +## Outcome + +The dependency refresh reduces the original audit from 23 package-level findings (8 moderate, 14 high, and 1 critical) to no production findings and four development-only findings in the documentation toolchain. + +The deployed SHACL Vue application now has a zero-finding production audit. +The exact package lock, application and library builds, test suite, documentation build, and a jsdom-backed Markdown sanitization regression all pass. + +Reviewed updates include Vue 3.5.41, Vite 7.3.6, Vitest 3.2.7, DOMPurify 3.4.13, Markdown-It 14.3.0, Mermaid 11.16.1, Happy DOM 20.11.2, YAML 2.9.0, and Vuetify 3.13.1. + +## Stable documentation boundary + +The four remaining findings are confined to the development documentation chain: + +```text +vitepress-plugin-mermaid 2.0.17 + -> vitepress 1.6.4 + -> vite 5.4.21 + -> esbuild 0.21.5 +``` + +VitePress 1.6.4 is the latest stable VitePress 1 release and declares Vite `^5.4.14`. +The latest Vite 5 release remains affected. +The patched Vite line is outside that supported range, while VitePress 2 is an alpha release that requires Vite 8 and is outside the Mermaid plugin's declared VitePress 1 peer range. +The package audit reports no supported automatic fix. + +Milestone 3 therefore does not force an unsupported dependency override or an alpha documentation migration. +Documentation packages are not present in the deployed browser runtime. +Human acceptance of this bounded development-only exception is tracked as M3-Q017 in the decision register. + +## Security review boundary + +Markdown preview rendering remains behind DOMPurify and now has a direct XSS regression. +The review also identified existing raw HTML rendering sites for deployment- and schema-authored presentation content. +Those sources are trusted, pinned inputs in this backend-free preview; making them remotely writable requires a separate sanitization design and is tracked as M3-Q016. + +Record-derived external actions must accept only HTTP or HTTPS destinations and open a new page without granting it control of the editor window. +The static patch-download work applies that narrow hardening without introducing a browser credential or hosted write service. + +The final nested commit includes both the dependency refresh and the static patch-download editor. +The wrapper commit pins that exact nested tree and uses a deterministic build timestamp derived from committed history. +Both nested and wrapper build labels use the stable value `pinned` rather than the checkout-local branch name, so a branch worktree and a detached recursive checkout produce the same bytes. +The wrapper also removes its generated runtime-plugin destination before every copy. +This prevents a second build from nesting the plugins beneath the first build's output, and makes fresh recursive checkouts match established worktrees. diff --git a/docs/milestone-3-editor.md b/docs/milestone-3-editor.md new file mode 100644 index 0000000..cdabcde --- /dev/null +++ b/docs/milestone-3-editor.md @@ -0,0 +1,56 @@ +# Milestone 3 static editing handoff + +Status: implementation complete; draft human review open + +## Outcome + +The Pages preview includes the production SHACL Vue interface under `/orinoco-lite-dev/edit/`, but it has no backend, authentication flow, service token, or GitHub credential. +It loads the pinned public records and schema from the same static artifact. + +An editor can change a record in the browser and download a deterministic RDF review bundle. +A checked-in local helper validates that bundle against the exact site commit and canonical YAML before showing a repository-relative diff. +Applying the diff remains an explicit local action. + +## Browser workflow + +1. Open a record in the Pages preview and select **Edit this record**. +2. Change the record in SHACL Vue and select **Save**. +This saves only in the browser's in-memory review queue; it does not send a request. +3. Open the download panel, select the intended records, and choose **Download review bundle**. +4. Preserve the downloaded JSON file unchanged for local validation. + +The bundle records the site commit, canonical PID, schema type, source path, source digest, and edited RDF. +It contains no credential and does not nominate or create a pull request. + +## Local validation and application + +Use a clean parent checkout with its site submodule at the bundle's recorded commit. +From the parent repository, inspect a bundle without changing files: + +```text +pixi run review-editor-bundle ~/Downloads/con-review-….json +``` + +The helper rejects an unknown PID, stale site commit or source digest, changed schema type or PID, path escape, invalid RDF, relationship/reference failure, ignored input, or dirty canonical checkout. +A successful dry run prints a unified diff with paths relative to the site repository. + +After reviewing that diff, apply the same validated update explicitly: + +```text +pixi run review-editor-bundle ~/Downloads/con-review-….json --apply +``` + +Review and commit the resulting YAML in the site repository. +A parent change then deliberately advances the site gitlink. +Milestone 3 does not automate either commit or open a component pull request. + +## Security and publication boundary + +- the browser performs only static `GET` and `HEAD` requests; +- the editor config disables service and token modes; +- shapes, records, class definitions, and deployment config are local, relative, digest-bound files; +- the Pages artifact contains no loopback service, German editor endpoint, token-shaped value, symlink, Git state, or `CNAME`; +- downloaded bundles are size- and record-count-bounded and fail closed; and +- authenticated branch creation, direct pull-request submission, hosted tokens, and production writes remain deferred. + +Playwright exercises the complete browser path at the project URL: it follows a generated edit link, loads Yaroslav Halchenko from static RDF, changes a field, downloads a bundle, verifies that the browser made no write request, and passes the bundle through the local dry-run validator without modifying canonical YAML. diff --git a/docs/milestone-3-pages.md b/docs/milestone-3-pages.md new file mode 100644 index 0000000..cb146fe --- /dev/null +++ b/docs/milestone-3-pages.md @@ -0,0 +1,134 @@ +# Milestone 3 GitHub Pages preview + +Status: implementation complete; draft human review open + +Target repository: `con/orinoco-lite-dev` + +Target project URL: `https://con.github.io/orinoco-lite-dev/` + +## Outcome + +Milestone 3 publishes the accepted CON static artifact as a GitHub Pages project site. +The deployment remains backend-free: the public website and the optional editor are ordinary files, no Dump Things process runs after the build, and no browser receives a service credential or GitHub token. + +This document is a new successor contract. +It does not rewrite the accepted Milestone 1 or Milestone 2 records. +The user's Milestone 3 authorization supersedes their former local-only boundary only for the parent branch, reachable submodule pins, a draft parent pull request, and the Pages preview. +It does not authorize DNS, a custom domain, production cutover, hosted write credentials, or pull requests in component repositories. + +## Artifact contract + +`pixi run build-pages` builds the CON profile, not the German upstream snapshot. +It uses the exact locked Pixi environment and recursive gitlinks, hydrates only manifest-declared assets through credential-free read-only HTTPS sources, and emits: + +```text +build/pages-preview/orinoco-lite-dev/ + .nojekyll + index.html + graph.js + graph.json + edit/ + index.html + config.json + editor-contract.json + record-sources.json + publication.json + ... +``` + +The Hugo base URL and graph, navigation, asset, and edit links all use `/orinoco-lite-dev/`. +The generated site points edit links at `/orinoco-lite-dev/edit/`; loopback URLs and the German editor URL are rejected from the uploaded artifact. + +`pixi run verify-pages` compiles the pinned SHACL Vue editor twice independently, requires those bundles to be byte-identical, then creates the complete Pages artifact twice and requires byte-identical file manifests. +`pixi run audit-pages` checks an existing artifact without rebuilding it. +`pixi run serve-pages` provides a local project-path preview at `http://127.0.0.1:8766/orinoco-lite-dev/`; that local origin is never embedded in the public files. + +## Static editing handoff + +The Pages builder requires a deterministic editor produced at `build/pages-editor`. +The bundle must contain `index.html`, `config.json`, and an `editor-contract.json` with these claims: + +```json +{ + "authentication": "none", + "backend": "none", + "mode": "patch-download", + "version": 1 +} +``` + +The parent copies the bundle under `edit/` and supplies `record-sources.json`. +The editor's `config.json` must also set `use_service` and `use_token` to `false`, `review_bundle_mode` to `patch-download`, and `review_bundle_catalog` to the relative `record-sources.json` path. +That catalog contains the exact public canonical YAML, its site-repository-relative path and digest, and the immutable site commit. +It is enough for a browser-only editor to generate a reviewable patch without an API call. +It intentionally excludes service endpoints, access tokens, automatic pushes, and automatic pull-request creation. + +A downloaded patch changes paths in the `centerforopenneuroscience.org` repository because that repository currently owns canonical YAML. +The parent repository contains only its gitlink. +A future one-click contribution flow therefore needs an explicit decision about the public contribution target and the subsequent parent-gitlink update. +The current milestone does not disguise that two-repository review boundary. + +## Actions security boundary + +`.github/workflows/con-pages-preview.yml` has separate build and deployment jobs: + +| Event | Build and upload artifact | Deploy shared Pages site | +| --- | --- | --- | +| Pull request to `main` | Yes | No | +| Push to `codex/milestone-3` | Yes | Yes, for the initial human-review preview | +| Push to `main` | Yes | Yes | +| Manual dispatch | Yes | Only when the `deploy` input is selected | + +The workflow gives pull-request code only `contents: read`. +The deployment job is structurally excluded from pull-request events and alone receives short-lived `pages: write` and OIDC permissions. +Checkout does not preserve credentials. +Every action is pinned to a full commit ID, Pixi itself is pinned to `0.73.0`, and the repository lock is mandatory. + +The Pages environment is a single shared preview, not a separate URL for each pull request. +Deployments are serialized and a newer deployment cancels an older one. +The temporary `codex/milestone-3` push trigger makes the draft pull request reviewable before merge; remove that trigger after deciding that `main` is the only publication branch. + +## Reachability and publication sequence + +GitHub Actions can check out only commits reachable from the configured public submodule URLs. +Publishing the parent branch therefore has this order: + +1. Run complete local Milestone 3 acceptance and freeze the site gitlink. +2. Push the exact site and any changed nested component commits to their configured read-only checkout repositories. +No component pull request is required by this milestone. +3. Confirm a disposable HTTPS recursive checkout resolves every gitlink. +4. Configure `con/orinoco-lite-dev` Pages to use GitHub Actions. +5. Push `codex/milestone-3` in `con/orinoco-lite-dev` and open one draft pull request to `main`. +6. Let that branch push publish the shared preview and record its workflow run and deployed URL in the Milestone 3 acceptance report. + +Pages is configured for GitHub Actions and the public preview is available at `https://con.github.io/orinoco-lite-dev/`. +The `github-pages` environment admits `main` and the temporary `codex/milestone-3` review branch. +The temporary branch policy and workflow trigger are review scaffolding, not a production branch policy. + +## Acceptance + +Publication is acceptable when all of the following hold: + +- the recursive HTTPS clone resolves the exact parent, site, theme, schema, projection, graph, editor, and asset pins; +- `test-pages` passes and two Pages builds are byte-identical; +- the artifact contains no symlinks, Git state, credentials, local URLs, German editor URL, or persistent-service dependency; +- homepage, people, projects, publications, graph resources, branding, and representative assets work under `/orinoco-lite-dev/`; +- the editor loads its static shapes and source record through the project path and downloads a patch that applies cleanly at the pinned site commit; +- pull-request execution cannot enter the deployment job; +- one branch deployment succeeds and its public result receives a human content review; and +- no listener, incoming edit, temporary annex remote, or token remains after local and browser acceptance. + +## Human decisions and clarifications + +These decisions are deliberately surfaced rather than hidden in deployment code: + +1. **Patch contribution repository.** Choose whether downloaded patches and eventual component pull requests target `con/centerforopenneuroscience.org` or the account mirror used to make the direct-upstream branch reachable. +2. **Publication branch after review.** Decide whether Pages should deploy only from `main` after Milestone 3 merges. +The recommended default is yes; the milestone branch trigger is a temporary preview bootstrap. +3. **Two-repository review.** Decide whether content remains canonical in the site repository, requiring a site change followed by a parent gitlink change, or moves into the parent in a later milestone. +This milestone preserves the proven site-repository ownership. +4. **Public source catalog.** Confirm that publishing canonical YAML verbatim in the editor catalog is acceptable. +It is already public website input, but the catalog makes bulk download convenient. +5. **Environment protection.** Choose the reviewers, if any, required by the `github-pages` environment before a manual or `main` deployment proceeds. + +Hosted authentication, automatic branch creation, automatic pull-request submission, custom domains, redirects, and production DNS remain deferred. diff --git a/docs/milestone-3.md b/docs/milestone-3.md new file mode 100644 index 0000000..3173b86 --- /dev/null +++ b/docs/milestone-3.md @@ -0,0 +1,138 @@ +# Milestone 3: publications and public preview + +Status: implementation complete; draft human review open + +Parent branch: `codex/milestone-3` + +Site branch: `codex/milestone-3` + +Accepted base: [`milestone-2-acceptance.md`](milestone-2-acceptance.md) + +Decision register: [`milestone-3-decisions.md`](milestone-3-decisions.md) + +Dependency review: [`milestone-3-dependencies.md`](milestone-3-dependencies.md) + +Static editor workflow: [`milestone-3-editor.md`](milestone-3-editor.md) + +## Outcome + +Milestone 3 makes the populated CON site representative enough for broad human review. +It adds the maintained CON Zotero publication feed through a repeatable API ingestion boundary, publishes a GitHub Pages project preview, and provides a credential-free static editing handoff that produces reviewable local input for a pull request. + +This milestone publishes a preview, not the production CON website. +It does not change DNS or the custom domain and does not add hosted authentication or a persistent metadata service. + +## Workstreams + +### Public Zotero API ingestion + +Use public Zotero group `6197458` as the maintained publication intake source. +The ingestion implementation must: + +- read through Zotero Web API v3 without credentials for the public library; +- follow API pagination and record the library version, item versions, collection membership, item keys, response metadata, and capture time; +- produce a deterministic, committed source snapshot and a separately generated candidate set; +- preserve source observations before duplicate resolution or semantic promotion; +- apply only reviewed exact creator mappings and fail unresolved identity closed; +- report excluded collections, duplicate DOI groups, unsupported item types, unresolved venues, missing identifiers, and unmodeled creators; +- promote reviewed publications and only semantically connected venue and reference records into the canonical site profile with provenance; and +- never write to Zotero as part of normal ingestion or CI. + +An API refresh proposes a source and canonical change. +It never updates the site implicitly. + +### GitHub Pages preview + +Publish the parent repository at the GitHub Pages project path `https://con.github.io/orinoco-lite-dev/` using a pinned GitHub Actions workflow. +The workflow must: + +- initialize every pinned submodule commit from a publicly readable GitHub remote; +- use the locked Pixi runtime and exact Node, Hugo, Python, and Git Annex dependencies; +- hydrate only the declared site asset manifest through read-only remotes; +- verify the committed projection and assembly digests; +- build and audit the project-path artifact; +- include the production SHACL Vue bundle and static editing inputs under `/orinoco-lite-dev/edit/`; +- upload one immutable Pages artifact and deploy it through the GitHub Pages environment; and +- avoid secrets, writable metadata services, custom domains, and production redirects. + +### Static editing handoff + +The preview editor may display and modify the public committed records in the browser. +It must operate without a service token and must not claim to submit a change directly. + +The supported handoff is: + +1. open a record through its preview edit link; +2. edit and validate it through SHACL Vue; +3. download an RDF review bundle; +4. apply that bundle in an authenticated local checkout through a checked-in validation helper; and +5. inspect the canonical YAML diff before committing or opening a pull request. + +The browser receives no GitHub credential. +Direct pull-request creation, OAuth, GitHub Apps, hosted tokens, and branch-protection policy remain a later milestone. + +### SHACL Vue dependency refresh + +Update the pinned pool UI and nested SHACL Vue dependencies, commit the exact lockfile, run the upstream unit/build suite and the parent Playwright suite, and record any advisory that cannot be removed without an unsupported behavior change. +Do not weaken browser security controls to make the update pass. + +## History and publication policy + +The accepted `codex/full-con-migration` parent and site branches do not move. +The Milestone 3 site branch drops and later regenerates the terminal projection commit around reviewed hand-authored batches. + +Required submodule commits may be pushed to their existing account or CON GitHub mirrors so a clean parent checkout can resolve each gitlink. +They do not receive pull requests in this milestone. + +After complete acceptance: + +1. push the exact submodule branches; +2. update and verify the parent gitlinks; +3. push parent `codex/milestone-3` to `con/orinoco-lite-dev`; +4. open one draft PR against parent `main`; +5. configure the parent repository's Pages source as GitHub Actions; and +6. verify the public preview and downloadable editing handoff. + +## Acceptance + +Milestone 3 is complete when: + +- a fresh public Zotero API capture is byte-reproducible after normalization and its source/library versions are recorded; +- every promoted publication passes source-schema validation, JSON-to-RDF-to-JSON round trips, reference closure, native relationship, and dangling-target checks; +- source observations, canonical decisions, exclusions, duplicates, and unresolved records have separate provenance; +- the terminal projection is regenerated twice byte-identically; +- root and project-path static builds are byte-identical on repetition and contain no German entity routes or graph nodes; +- the production SHACL Vue build has a committed lockfile and the dependency audit has no unreviewed high or critical finding; +- Playwright proves that a preview record can be edited without a token and downloaded as RDF without a network write; +- the local application helper rejects unrelated, unknown, invalid, and stale bundles and produces only the intended canonical YAML changes; +- a disposable public clone resolves every pushed gitlink and reproduces the Pages artifact; +- one draft parent PR exists, no submodule PR exists, and the Pages preview is publicly reachable; and +- all required human decisions are enumerated in the decision register and PR description. + +## Deferred + +Milestone 3 does not include: + +- automatic or authenticated browser creation of GitHub branches or pull requests; +- OAuth, GitHub Apps, hosted write tokens, or production authorization; +- production-domain cutover, DNS, redirects, or replacement of the legacy site; +- automatic writes to Zotero; +- a persistent public Dump Things service; +- final resolution of every creator, venue, asset, or hidden-person ambiguity; +- grants, CVs, annual reports, or secondary projections; or +- a broad upstream template fork. + +## Implementation checkpoint + +The reviewed public capture is Zotero library version 451 with 197 top-level items in five collections. +The deterministic transform currently yields 126 publications, 20 venue candidates, four datasets, and three instruments. +The site promotes all 126 publications, one reviewed topic, and the required bibliographic reference closure. + +The 20 venue candidates remain in the ingestion evidence rather than becoming disconnected site records. +The source relates them through a generic publishing placeholder rather than a canonical native activity. +Typed ISSNs remain on the publications, and the unresolved activity/venue decision remains visible in the decision register. + +The candidate site projection contains 186 canonical records, 13 reference records, 186 graph nodes, 467 native edges, and 185 rendered record pages. + +The implementation, publication evidence, and bounded debt are recorded in [`milestone-3-acceptance.md`](milestone-3-acceptance.md). +The draft pull request and public preview are review surfaces; this status does not imply final human content approval or a production cutover. diff --git a/docs/orinoco-lite-plan.md b/docs/orinoco-lite-plan.md index 4e8d17d..03bd126 100644 --- a/docs/orinoco-lite-plan.md +++ b/docs/orinoco-lite-plan.md @@ -1,310 +1,130 @@ # Orinoco Lite execution plan -Status: active +Status: active — local full CON migration ## Outcome -Enable a lab to create one GitHub repository, add structured research metadata -and editorial content, and deploy a static lab website through GitHub Pages. -The lab should not need to understand or operate the underlying Orinoco -services. +The long-term Orinoco Lite outcome is a self-contained lab repository with human-editable YAML metadata, editorial content, and a deterministic static website. +Git pull requests may eventually provide the review and publication boundary without requiring a continuously running metadata service. -Canonical metadata is human-editable YAML in Git. GitHub pull requests provide -the authentication, review, and publication boundary. The same records may -later support websites, graphs, grants, CVs, annual reports, and other -projections. +The accepted **clean migration** proved the upstream-compatible metadata, isolated collection, editor, static deployment, browser acceptance, and rebase strategy on a connected CON vertical slice. +Its frozen contract remains in [`docs/clean-migration.md`](clean-migration.md). -## Current repository map +The active **full CON migration** expands that architecture into a populated CON website. +Its implementation contract is [`docs/full-con-migration.md`](full-con-migration.md). -| Repository | Current role | +## Repository roles + +| Repository or branch | Current role | | --- | --- | -| [`con/orinoco-lite-dev`](https://github.com/con/orinoco-lite-dev) | Development workspace, architecture, component pins, and integration coordination | -| [`con/www-from-model`](https://github.com/con/www-from-model) | Clean GitHub mirror of upstream history and staging point for reusable changes | -| [`www/www-from-model`](https://hub.psychoinformatics.de/www/www-from-model) | Upstream metadata-driven Hugo website | -| `centerforopenneuroscience.org` | Final CON repository for canonical metadata, content, presentation, and deployment | -| `dump-research-info` | Source of reviewed CON metadata and migration/provenance logic | -| Orinoco repositories | Upstream schemas, Dump Things, `qri`, graph, UI, and enrichment components | -| `con/orinoco-lite-action` | Future released build and deployment implementation; not yet extracted | -| `con/orinoco-lite-template` | Future minimal lab starter; not yet created | - -The coordination repository currently pins 24 independently usable component -repositories under `submodules/`. Their present location does not imply that -all are runtime dependencies. Directory reorganization is not required before -the first website slice. - -## Repository and branch policy - -The existing `centerforopenneuroscience.org` repository remains the durable CON -website repository. Preserve its current source history as: - -- `legacy-site`, a branch at the pre-Orinoco Lite site tip; and -- `legacy-site-2026-07-31`, an immutable baseline tag. - -Create `orinoco-lite` from the existing CON `master` branch. This preserves -normal CON ancestry and permits ordinary fork-based pull requests. Import only -the useful `www-from-model` scaffolding in focused, attributed commits. - -The local `submodules/www-from-model` repository keeps: - -- `origin`: `https://github.com/con/www-from-model.git`; -- `upstream`: `https://hub.psychoinformatics.de/www/www-from-model.git`; and -- `main`: a clean mirror of `upstream/main`. - -Track the last reviewed `www-from-model` commit explicitly. Review subsequent -upstream changes against that commit, then copy, adapt, or cherry-pick only the -parts that remain useful. The CON branch is never rebased onto the complete -upstream website history. - -Keep changes separated where practical: - -- CON metadata, existing content, assets, theme changes, and adopted scaffold - code belong on `orinoco-lite`. -- Generally useful fixes belong on focused branches in `con/www-from-model` - and should be offered upstream narrowly. -- Released Orinoco Lite behavior will eventually be extracted into a separate, - versioned action rather than maintained as permanent CON-specific patches. - -The parent repository pins explicit component commits. Updating an upstream -reference must not silently change a released lab build. - -## Architectural decisions - -- A lab starts from a small GitHub template, not a fork of this coordination - repository. -- The default topology is one self-contained repository containing metadata, - website content, configuration, and deployment. -- A separate metadata repository is an optional later topology for cases with - distinct permissions, release schedules, or independently managed consumers. -- The public site is a deterministic static projection and requires no - continuously running metadata backend. -- Dump Things may run locally and ephemerally in CI to preserve upstream - validation and projection behavior. -- JSONL is an internal adapter when required by `qri`, not a second canonical - format. -- RDF is generated only when an identified consumer requires it. -- Direct Git editing and pull requests are the complete initial editing path. -- GitHub Issue Forms, SHACL-vue, a GitHub App, and an OAuth broker are possible - later improvements, not first-milestone dependencies. -- GitHub Pages is the initial preview and deployment target. - -## First milestone: CON vertical slice - -### Goal - -Produce a GitHub Pages preview from one small, connected set of real CON -records using as much of the upstream `www-from-model` pipeline as practical. -This milestone establishes the actual metadata, generator, theme, and workflow -boundaries before they are generalized. - -### Minimum data slice - -Include: - -- the Center for Open Neuroscience organization; -- one person; -- one project; -- publications -- the relationships connecting those records; -- one or more representative depictions or assets; and -- enough editorial content to evaluate the homepage and primary navigation. - -Use records that already have reviewed evidence in `dump-research-info` and -content or assets already present in `centerforopenneuroscience.org`. - -### Execution sequence - -1. Fully hydrate the legacy `centerforopenneuroscience.org` history and preserve - its current tip as `legacy-site` and `legacy-site-2026-07-31`. -2. Create `orinoco-lite` from the existing CON `master`, push it to a writable - fork, and open a draft pull request without changing production deployment. -3. Capture the current deployed site output, representative screenshots, - public URL inventory, and essential visual design values for comparison. -4. Identify the exact metadata collection, schema release, templates, scripts, - and build entry points worth adopting from `www-from-model`. -5. Import the minimum useful Hugo, Congo, projection, and build scaffolding in - separate commits with explicit upstream provenance. -6. Select the minimum connected CON records and their source evidence. -7. Transform those records into individual upstream-compatible YAML files on - `orinoco-lite`. -8. Preserve the corresponding editorial content, assets, URLs, and design - values from the existing CON tree. -9. Adapt the build so required Dump Things behavior runs ephemerally during CI - rather than depending on a dedicated service. -10. Use `qri` and Hugo to produce the static website. -11. Add a repository-local GitHub Actions workflow that publishes a GitHub - Pages preview from the prototype branch. -12. Record necessary divergence and keep reusable fixes isolated for possible - contribution. - -The production build must consume the combined candidate tree. It must not -continually join independently changing data from `dump-research-info` and -content from `centerforopenneuroscience.org`. - -### Exit criteria - -- One canonical metadata change deterministically changes the preview site. -- The selected records validate through the upstream Orinoco path. -- Invalid records stop publication. -- The website builds without contacting a persistent metadata service. -- The preview contains the selected organization, person, project, output, - relationships, and representative assets. -- The build uses only files in the candidate lab repository plus pinned build - dependencies. -- CON-specific and potentially reusable changes remain distinguishable. -- No production DNS, domain, or existing-site deployment is changed. - -## Metadata and build contract - -- Store approved entities as individual YAML records following the filesystem - and schema conventions selected from upstream Orinoco. -- Use `.dumpthings.yaml` collection configuration where required by that - upstream layout. -- Preserve source observations, retrieval details, candidates, - reconciliation decisions, and reviewed additions separately from approved - public records. -- Maintain stable identifiers and explicit relationships during migration. -- Pin a compatible schema and toolchain for every reproducible build. -- Run Dump Things ephemerally in CI using the upstream Orinoco validation path. - Direct LinkML validation may provide an earlier, faster check, but it is not - the sole publication gate unless its scope is shown to match the required - Dump Things checks. -- Convert validated records to JSONL transiently when `qri` requires its stream - interface. -- Generate RDF only for a named editor, graph, or interoperability consumer. -- Treat generated Hugo content, caches, JSONL, RDF, and pages as build - artifacts rather than canonical records. -- Keep editorial Markdown, theme overrides, media, redirects, and site - configuration with the lab website. - -## Final lab repository organization - -The exact class directories will follow the selected upstream collection, but -the stable responsibility boundaries are: - -```text -metadata/ canonical YAML records and collection configuration -content/ human-authored editorial content -assets/ branding, logos, fonts, and processed assets -static/ static files copied into the site -layouts/ CON-specific Hugo overrides -config/ Hugo, Congo, navigation, and color configuration -provenance/ import manifest and legacy URL mapping -.github/workflows/ validation, preview, and Pages deployment -README.md editing and deployment instructions -UPSTREAM.md source repository, base commit, and sync policy -``` - -Generated metadata pages and projection files should be produced in a build -workspace or ignored generated directory, not committed alongside canonical -records. - -## Explicit first-milestone non-goals - -- Full migration of all CON records. -- Production cutover or DNS changes. -- Pixel-level parity with the existing CON website. -- Extraction of `orinoco-lite-action`. -- Creation of `orinoco-lite-template` or Copier prompts. -- A separate canonical metadata repository. -- GitHub App, OAuth broker, or direct graphical writes. -- Published JSONL or RDF without an identified consumer. -- Grant, CV, annual-report, or other secondary projections. -- Reorganization or wholesale updating of every tracked submodule. -- Broad upstream refactoring. - -## Later phases - -### 2. Complete CON migration - -- Migrate all accepted records from `dump-research-info` into canonical YAML. -- Preserve source snapshots, evidence, review decisions, merge policy, and - identifiers. -- Compare record counts, persistent identifiers, relationships, and approved - assets before retiring any migration path. -- Keep `dump-research-info` available as migration history until parity is - accepted. - -Exit: the candidate lab repository is the single canonical source for accepted -CON metadata and website content. - -### 3. Website completion and cutover - -- Complete content and presentation work based on the proven vertical slice. -- Account for existing public URLs, redirects, assets, attribution, and - accessibility. -- Establish branch protection, reviewers, Pages ownership, and deployment - permissions. -- Review the static preview before changing the production domain. - -Exit: the candidate can replace the existing CON deployment without losing -required content, URLs, or provenance. - -### 4. Reusable action and lab template - -- Extract validation, projection, Hugo build, and Pages deployment from the - working CON implementation into `con/orinoco-lite-action`. -- Expose one tagged reusable workflow with conventional paths and pinned - dependencies. -- Obtain Pages URL and base-path information from GitHub Pages configuration - rather than requiring users to calculate it. -- Create `con/orinoco-lite-template` with the canonical directory skeleton, - example records, lab configuration, and a short caller workflow. -- Add Copier only if prompted initialization materially improves setup after - the plain GitHub template works. -- Test the release in a new independent lab repository. - -Exit: a lab can replace example metadata and deploy without understanding the -Orinoco toolchain. - -### 5. Editing and additional projections - -- Document direct file and pull-request editing first. -- Evaluate Issue Forms for bounded metadata additions. -- Configure SHACL-vue for validation and export before introducing direct - writes. -- Consider a GitHub App and stateless OAuth broker only if the editing benefit - justifies operating an external endpoint. -- Add RDF, grants, CVs, annual reports, and other projections only for concrete - consumers. - -Exit: additional interfaces preserve the same canonical records and Git review -boundary. - -## Upstream synchronization and contribution policy - -- Check upstream component heads on a deliberate schedule. -- Never auto-merge upstream updates. -- Summarize relevant component changes and test candidate pins before release. -- Ensure action tags pin reproducible dependency versions. -- Develop reusable fixes on focused branches in the relevant repository. -- Submit only narrow, generally useful changes upstream. -- Keep CON-specific policy, metadata, content, and presentation downstream. - -## Deferred decisions - -These do not block the first vertical slice: - -- final ownership and naming of the action and template repositories; -- the public product documentation entry point; -- the exact template configuration and optional Copier questions; -- whether guided editing begins with Issue Forms or SHACL-vue export; -- whether an external OAuth broker is ever acceptable; -- criteria for splitting metadata into a separate repository; -- published RDF or JSONL contracts; -- metadata licensing, CODEOWNERS, and long-term review policy; -- production Pages, DNS, and domain ownership; and -- the final pull-request preview experience. - -Resolve each decision immediately before the phase that depends on it. Do not -block the vertical slice waiting for speculative answers. - -## Handoff for the next thread - -Implement only the first milestone above. The primary implementation repository is -`submodules/centerforopenneuroscience.org` on `orinoco-lite`. Preserve -`legacy-site` for content and visual comparison, push implementation commits to -the `leej3` fork. Use `submodules/www-from-model` as the clean upstream mirror and -`submodules/dump-research-info` only as a migration input. Keep track of your progress in docs/milestone-1-progress.md. - -Do not begin by generalizing the action, creating more architecture documents, -or reorganizing all submodules. First prove one real record-to-website path. +| Parent `codex/clean-migration` | Immutable accepted coordination checkpoint | +| CON site `codex/clean-migration` | Immutable accepted two-commit site checkpoint | +| Parent `codex/full-con-migration` | Active local tooling, tests, policy, and deliberate site gitlink | +| CON site `codex/full-con-migration` | Active direct-upstream content-migration successor | +| CON site `master` and preservation refs | Legacy production history and migration evidence; unchanged | +| CON site `orinoco-lite` | Completed legacy-derived vertical slice; unchanged evidence | +| `www-from-model` `main` | Clean mirror of `upstream/main` and source of reviewed bases | +| `dump-research-info` | Structured migration evidence; never a normal build-time source | +| Orinoco submodules | Explicitly pinned schema, service, qri, UI, and graph components | + +The full-migration site branch uses reviewed upstream commit `a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8`. +The single upstream change after the accepted `5b401e0` base is a reviewed Forgejo CI path-to-URL correction; it does not alter presentation or generated content. + +## History and authority policy + +Direct upstream ancestry remains an intentional, narrow exception for the accepted clean-migration site branch and its full-migration successor. +It does not change the ancestry or production status of `master`, `legacy-site`, `orinoco-lite`, or preservation refs. + +The clean-migration branches do not move. +The active site successor uses: + +1. the accepted foundation replayed onto the reviewed upstream base; +2. ordinary focused commits for hand-authored profile and content batches; and +3. one terminal regenerable projection commit containing generated outputs only. + +Clean-site YAML is the sole canonical metadata authority. +The legacy website and `dump-research-info` may supply evidence for a reviewed migration decision, but neither participates in a normal build. +Editorial Markdown, configuration, and declared assets in the clean-site tree are similarly authoritative for the static presentation. + +## Active milestones + +### 1. Generalize the vertical-slice contracts + +Replace exact vertical-slice lists in projection, stack, graph, editor-link, asset, and test tooling with one executable profile manifest. +Preserve the accepted person, project, publication, instrument, organization, and homepage root as representative regression assertions. + +The generalized contract must drive canonical inventory, renderable classes, routes, visibility and ordering, reference closure, native relationship integrity, assets, and acceptance expectations. +Split metadata-projection invalidation from static-site assembly invalidation so editorial-only changes do not force a metadata reprojection. + +This milestone is complete only when the generalized implementation reproduces the accepted slice deterministically and passes the full unit, build, service, and Playwright suite. + +### 2. Restore legacy-equivalent public coverage + +Migrate the reviewed public experience in coherent batches, beginning with the legacy evidence inventory of 33 visible people and 23 featured projects. +Then restore the homepage, navigation, contact/support and other essential editorial pages, CON branding, portraits, and project imagery. + +The counts are reconciliation baselines, not quotas. +Record reviewed merges, exclusions, additions, and unresolved identities. +Preserve public ordering and editorial intent where supported by evidence, while using upstream information architecture and avoiding a broad template fork or pixel-level theme rewrite. + +Every batch includes provenance, native relationships, reference closure, asset status, and acceptance expectations. +Generated files remain isolated in the terminal projection commit. + +## Contracts retained from clean migration + +The successor keeps these proven boundaries: + +- source-schema validation with exact `dlthings:*` CURIEs; +- canonical `xyzrins:.` project root and a distinct CON organization record; +- four isolated local service collections; +- `con-public` as the sole CON projection source; +- `con-protected` as the anonymous-read, token-limited local edit boundary; +- explicit-only upstream reference interfaces; +- deterministic root and project-path static builds; +- content-derived graph cache identities; +- Chromium and WebKit browser acceptance; and +- no persistent metadata service for the deployed static artifact. + +Expected inventory and graph totals now derive from the reviewed profile manifest rather than fixed slice counts. +Tests must continue to detect German data leakage, stale generated products, invalid native targets, unsafe editor writes, broken assets, and process or credential residue. + +## Upstream synchronization + +Upstream changes are reviewed before use. +Preserve the current successor tip, inspect the candidate upstream range, remove the terminal generated commit from the replay, rebase hand-authored commits, regenerate one terminal projection commit, inspect `range-diff` and both content digests, and run complete local acceptance. +Update the parent site gitlink only after all checks pass. + +Never rewrite the accepted clean-migration branches during this drill. +Prefer adapting CON profile/content to a new upstream convention over adding a compatibility layer. + +## Local-only boundary + +This phase may create local branches, safety refs, generated state, and local test services. +It may read remotes to review upstream or retrieve already identified public assets. + +It must not push, open or update pull requests, publish Pages, change repository settings, alter DNS or production hosting, write to public metadata services, or store credentials in a repository. + +## Deferred work + +Do not expand the active milestones to include: + +- the broader Zotero collection or bulk publication migration; +- GitHub Pages, previews, production cutover, DNS, redirects, or custom domains; +- pull-request editing, GitHub Apps, OAuth, hosted authentication, or branch protection; +- a persistent hosted metadata service; +- experimental LinkML/schema branches or full-URI type support; +- grants, CVs, annual reports, and secondary projections; +- pixel-level legacy parity or a broad upstream template fork; +- a separate metadata repository or published RDF/JSONL contract; or +- unrelated upstream contribution work. + +Revisit Zotero after the people, project, editorial, and asset reconciliation is accepted. +Treat deployment and pull-request editing as later separately authorized phases. + +## Handoff + +Work only on the parent and site `codex/full-con-migration` branches. +Preserve all accepted and legacy refs, keep clean YAML authoritative, complete contract generalization before bulk migration, and retain a single regenerable terminal projection commit. + +Run complete local acceptance and the rebase drill before changing the parent site gitlink. +Do not push or deploy. diff --git a/docs/upstream-psychoinformatics-trial.md b/docs/upstream-psychoinformatics-trial.md new file mode 100644 index 0000000..65a2dcc --- /dev/null +++ b/docs/upstream-psychoinformatics-trial.md @@ -0,0 +1,411 @@ +# Upstream Psychoinformatics reproduction and Pages trial + +Status: local reproduction and deployment prototype complete; no production site, DNS, or GitHub Pages setting changed + +Date: 2026-08-10 + +Branch: `codex/upstream-psychoinformatics-trial`, created from parent `main` at `47bdf2f396e462d6622a166d2ba6c29f6a273b7c` + +## Executive conclusion + +The current Psychoinformatics website can be rebuilt and deployed as a static site without a running Dump Things service. +The exact current upstream commit builds deterministically with Hugo Extended 0.154.5 from its committed Markdown, graph, theme, and annexed assets. +Two builds each produced 1,973 Hugo pages and 2,058 files, and a fresh checkout from the repaired `con` mirror produced the same artifact. + +That result has an important limit: it reproduces the **published projection**, not the metadata system that generated it. +Regenerating or refreshing the projection still calls the live Psychoinformatics pool and a moving, unlocked `dtc`/`qri` toolchain. +The public pool's canonical records are not present in the checked-out repositories. +Therefore: + +- a frozen upstream website snapshot is deployable without Dump Things; +- a fresh metadata projection is not reproducible offline from this repository set; and +- the CON Git-native approach is still needed if the lab repository is to own canonical metadata and validate each publication from source. + +A standard GitHub Pages **project** URL adds a separate presentation issue. +Hugo honors the configured base URL for ordinary site links, but upstream templates, the compiled graph renderer, `graph.json`, and the web manifest contain root-absolute paths. +At a URL such as `https://con.github.io/orinoco-lite-dev/`, the graph, some controls, and installable-app icons therefore request the domain root and fail. +The trial includes a small generated-artifact adapter that leaves upstream source untouched, fixes those paths, and rejects any remaining escape. +Browser testing confirmed the homepage graph and a representative record graph at the project path with no console errors or warnings. + +The remaining external static-build dependency is Git Annex storage. +The `con/www-from-model` Git mirror is now current and carries the upstream annex metadata branch, but GitHub is not an annex object store. +A fresh GitHub-only clone could retrieve 25 annexed paths from their original web URLs but could not retrieve 14 required paths, including `graph.js`, `graph.json`, core CSS, branding, and favicons. +Adding the upstream repository as an explicit annex object remote hydrated all 39 paths. +A build can therefore run today, but a fully self-contained GitHub reproduction would need to mirror all 38 unique annex objects (55,916,505 bytes), not merely the Git refs. + +## Scope and isolation + +This investigation is isolated from the existing Orinoco Lite and LinkML trials: + +- the parent branch starts directly from `main`; +- its worktree is `/Users/johnlee/code/CON/orinoco-upstream-trial`; +- the dirty `orinoco-lite-dev` worktree and its submodule states were not changed; +- the site submodule is pinned to current upstream `5b401e0c478a4409442b3a8a285bd3efd5d30e05`; and +- only a generated deployment artifact is adapted for a Pages project path. +No upstream website source or generated content was edited. + +The phrase "German patches" is interpreted here as the LinkML patches and runtime monkeypatches carried by the upstream Psychoinformatics/Orinoco repositories. +That interpretation did not block the work; the exact patch inventory is below. + +## Git and mirror state + +The parent had pinned `www-from-model` at `6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce`. +Current upstream is three commits later: + +| Commit | Change | +| --- | --- | +| `10087fa5` | Metadata refresh touching 747 files and expanding publications from 115 to 846 records | +| `e50f88f` | A subsequent generated graph refresh | +| `5b401e0c` | Changes one Register Depictions preparation action from a deleted local path to a remote URL | + +The net snapshot change is large but mostly unrelated to LinkML: publications increase by 731, graph size increases from 307 nodes/882 edges to 1,038 nodes/2,148 edges, and Hugo output increases from 491 pages/576 files to 1,973 pages/2,058 files. + +The account-owned `https://github.com/leej3/www-from-model.git` mirror carries the upstream site refs and the deployment branch's nested account URL. +It was populated without rewriting the upstream history: + +| Ref | Current value | +| --- | --- | +| `refs/heads/main` | `6c8b9a5b7260dc20dfe1453dd863b353e8f90f06` | +| `refs/heads/git-annex` | `010ca44f751d2ab60b9d4ad58c5931d1804e3c9e` | + +The parent `.gitmodules` entry now uses the `leej3` mirror. +Local site checkouts use `origin` for that mirror and `upstream` for the Psychoinformatics source. +The Hugo theme remains the exact nested gitlink `3623fa505ee42fee899844d94a4ff7f5a1ae9096` from the upstream site. + +A live comparison of all 24 parent gitlinks found 20 exactly at their configured/default remote heads. Besides `www-from-model`, the exceptions are the deliberately unpublished CON site pin, the deleted/divergent `dump-research-info` branch, and a six-commit-stale `things-enrichment-tools` pin that is not used by the website workflows. +The complete table is in `provenance/upstream-psychoinformatics/submodule-inventory.tsv`. +The only intentional `con` URL remaining in the parent is the migration-input `dump-research-info` repository. + +## Account-owned recursive mirrors + +The deployment branch now uses public `leej3` GitHub mirrors for every top-level submodule except `submodules/dump-research-info`. +That repository is intentionally left at `github.com/con/dump-research-info` because it is a CON-owned migration input. +The account mirrors preserve the pinned commits and the fetched upstream branches; existing `leej3` repositories were retained without deleting their unrelated branches. +Nested dependencies are covered as well: the pool UI points to `leej3/shacl-vue`, the website and CON site themes point to `leej3/congo`, and `tools` points to `leej3/datalad-concepts`. + +The account mirror set is deliberately separate from the parent repository: `orinoco-lite-dev` remains the only repository in this workflow that is not under `leej3`. +A fresh recursive checkout therefore needs only public GitHub URLs, while `dump-research-info` remains visibly attributable to CON in the parent `.gitmodules` file and inventory. + +## What is actually required + +The dependency boundary differs sharply by operation: + +| Operation | Required inputs | Dump Things / LinkML involvement | +| --- | --- | --- | +| Rebuild committed website | `www-from-model`, Congo, 39 annexed paths, Hugo Extended 0.154.5 | None | +| Deploy committed website | Rebuilt static artifact and a static host | None | +| Refresh pages and graph from current pool | Live `https://pool.psychoinformatics.de/api`, `dtc`, `qri`, `pool2graph.py`, Jinja templates, Annex, Forgejo actions | The remote pool service has already validated records; the refresh client itself uses no local LinkML model generation | +| Register/update depictions | Live pool, enrichment script downloaded from moving `main`, remote media URLs, Annex, deposit action | No local schema generation, but several unpinned remote services and actions | +| Recreate the pool from canonical records | Canonical source records, schema, LinkML, Dump Things service/client, storage and authorization configuration | Full stack required; the source record snapshot is missing here | +| Build Orinoco Lite from Git-owned YAML | Git records, pinned schema/toolchain, ephemeral local Dump Things, qri, templates, Hugo | Full validation/projection stack runs temporarily, then is discarded | + +For the static deployment, only one parent submodule is needed: `submodules/www-from-model`, plus its nested Congo submodule. +The other Orinoco submodules are diagnostic or refresh inputs, not static runtime dependencies. + +## Local reproduction + +The repeatable entry point is: + +```bash +BASE_URL=http://127.0.0.1:1313/ \ + tools/build_upstream_site.sh +python3 -m http.server 1313 --directory build/upstream-psychoinformatics +``` + +The script: + +1. checks out only the pinned upstream site and Congo theme; +2. pins annex metadata at `010ca44f...`; +3. retrieves annex content, using the upstream hub explicitly for objects that are not available from ordinary web URLs; +4. requires Hugo Extended 0.154.5; +5. builds the unchanged upstream source with the requested base URL; and +6. adapts and audits only the generated artifact when the URL has a non-root path. + +The full measured result is in `provenance/upstream-psychoinformatics/baseline.yaml`. +Important values are: + +| Measure | Result | +| --- | ---: | +| Committed record bundles | 15 datasets, 25 instruments, 5 objectives, 27 persons, 15 projects, 846 publications, 19 topics | +| Graph | 1,038 nodes, 2,148 edges | +| Hugo output | 1,973 pages, 2,058 files | +| Exact-build output size | 102,474,096 bytes | +| Repeatability | Two byte-identical sorted content manifests | +| Live pool requests during build | 0 | +| Dump Things processes during build | 0 | + +The manifest digest records a sorted list of each relative output path and its SHA-256, then hashes that list. +It is a comparison identifier for this trial, not an upstream release checksum. + +## Browser and GitHub Pages result + +Three browser cases were checked: + +1. An exact `hugo --minify` build inherits upstream's production `baseURL=https://www-draft.psychoinformatics.de`. +Its locally served HTML tries to load theme assets from that production host. +This is expected configuration behavior, not a missing build file. +2. A root-local build with `--baseURL http://127.0.0.1:8767/` loaded all theme and graph assets locally. +The homepage displayed seven Sigma canvases with no console error or warning. +3. A project-path build with `--baseURL http://127.0.0.1:8766/orinoco-lite-dev/` loaded normal Hugo navigation but initially requested `/graph.js` and `/explore` at the domain root. +After the artifact adapter, the homepage and a representative dataset page each displayed seven graph canvases, and all tested navigation stayed below `/orinoco-lite-dev/` with no console error or warning. + +Before adaptation the project-path artifact contained: + +| Root-path source | Count | +| --- | ---: | +| HTML `href`/`src` references | 974 | +| Compiled graph fetch for `/graph.json` | 1 | +| Graph node navigation URLs | 998 | +| Web-manifest icon URLs | 2 | + +The adapter changed 962 generated HTML files, 974 HTML URLs, one graph fetch, 998 graph node URLs, and two web-manifest icon URLs. +A second pass made zero changes, the path-leak audit reported zero findings, and two clean build/adapt runs had the same manifest digest `a58fee0aec0d8725c72b7d26068dc340b742494c4520480f80555e1dc6246c14`. +A fresh checkout from the repaired GitHub mirror produced that same digest. + +The adapter is intentionally downstream deployment glue. +A preferable general upstream change would make templates use Hugo `relURL`/`RelPermalink`, pass a base-aware graph-data URL into the renderer, and make `pool2graph.py` emit site-relative or explicitly based navigation values. +Until such a change is accepted, changing only the generated artifact avoids maintaining a fork of the upstream website source. + +The repository has no GitHub Pages site configured at present: the GitHub API returns `404` for the parent, site mirror, and legacy CON site repositories. +The included manual workflow is ready for a Pages site whose source is set to GitHub Actions, but this trial deliberately did not enable Pages or publish an external preview. +This avoids claiming a shared Pages environment or changing any production domain before review. + +## Local SHACL Vue and service-backed editor + +The upstream edit footer originally hard-codes `https://pool.psychoinformatics.de/ui/`. +Pixi-controlled local builds set `SHACL_VUE_URL=http://127.0.0.1:3000/`; the generated-artifact adapter rewrites the 953 edit links while preserving each `sh:NodeShape`, `pid`, and `edit=true` query parameter. +The production Pages workflow leaves the upstream URL as its default, so this local stack does not alter the static deployment. + +The editor is now deployed with the upstream service architecture locally. +The one-command entry point is: + +```bash +pixi run serve +``` + +Its Pixi dependencies perform the recursive checkout, upstream snapshot preparation, Hugo build, and pool UI build. +The `serve_local_stack.sh` supervisor then starts Dump Things, seeds both local collections, starts the git-annex and SHACL Vue services, checks their contracts, and serves the generated site. +All child logs are written below `build/local-stack/logs/`; Ctrl-C stops the complete stack. +The individual service tasks below remain useful when debugging a single component, while `pixi run serve-static` serves only the generated Hugo output. + +The underlying tasks are: + +1. `pixi run prepare-local-stack` downloads the public `Thing` collection from the upstream pool API into ignored `build/local-stack` state. +The measured snapshot contains 4,978 records. +`pixi run refresh-local-pool` explicitly refreshes that snapshot. +2. `pixi run serve-dump-things` runs the pinned Dump Things service (`9f101d97c7f15d491f602db5a9c33ad9a19ad8bf`) against a generated local configuration and the pinned Things Schemas YAML. +`pixi run seed-local-pool` loads the snapshot into both local `public` and `protected` collections. +3. `pixi run serve-git-annex` exposes an actual local git-annex repository at `http://127.0.0.1:8122/git-annex`, using the same p2p-over-HTTP path shape as the upstream uploader. +Uploads are keyed and stored by git-annex; they are not written to a demo-data directory. +4. `pixi run serve-shacl-vue` builds and serves the tracked `submodules/pool.psychoinformatics.de-ui` deployment branch at port 3000. +Its nested SHACL Vue checkout is pinned to `d5790a4431f7773a2e29fbb0d26e542ed0311ec5`, where the compatibility fix is a normal submodule commit. +The deployment branch tracks its local service configuration, generated schema assets, and local git-annex target as reviewable commits. + +The pool UI uses `use_service: true` and `use_token: true`, with read/write URLs pointing to the local Dump Things `public`/`protected` collections. +Its `config_default_xyzri.yaml` keeps `data_url` empty so no bundled RDF records are mistaken for the source of truth; schema, shape, and prefix assets are served from the tracked deployment checkout. +Direct local API calls use the same `/record` and `/records` route forms consumed upstream. + +Browser verification opened a generated dataset edit link, fetched the real record through local Dump Things, changed its title, and submitted it. +The protected incoming view contained the new title while protected curated and public remained at the old title, demonstrating the upstream curation boundary. +`pixi run check-local-stack` checks these service, record, schema, and configuration contracts without requiring a browser. + +This is a faithful local deployment of the service interactions, with two explicit scope limits: the local protected collection is seeded from the public upstream snapshot rather than private records requiring credentials, and the local git-annex repository is a single-process development service, not a production Forgejo host. +Neither limitation is hidden behind bundled demo data or a disabled backend; both are visible in the generated runtime configuration and Git history. + +## Annex boundary and GitHub-only reproducibility + +The upstream source has 39 annexed worktree paths representing 38 unique keys and 55,916,505 hydrated bytes. +There is no Git LFS configuration. + +A fresh clone from `leej3/www-from-model` showed two availability classes: + +- 25 paths can currently be fetched through URLs registered in annex; and +- 14 paths (1,276,022 bytes) are available only from the upstream annex repository. +These include the graph bundle/data and first-party visual assets. + +The exact paths and keys are recorded in `baseline.yaml`. +The Pages workflow adds the upstream repository as an object remote and pins the annex metadata commit, so it is operational without Dump Things. +It is not fully independent of Psychoinformatics infrastructure. + +To make the build GitHub-only and durable, mirror **all** 38 unique object contents to an immutable GitHub-compatible store and record content hashes. +Reasonable options are a release-asset bundle, a dedicated Git LFS asset repository, or normal Git for the small first-party files plus immutable release storage for the 48 MB depiction. +Copying only the 14 hub-only objects would make today's build work but would leave the 25 URL-backed objects exposed to origin removal or content drift. + +## Static snapshot quality findings + +The current upstream snapshot has issues unrelated to the Pages base path: + +- 26 of 998 graph navigation URLs have no generated target page: 8 organizations, 8 persons, and 10 projects. +Some graph entities are retained as relationship context even though the page-refresh queries intentionally filter which records receive pages. +One person also has an obsolete slug override (`/persons/yaroslav-halchenko`) while the generated page uses an ORCID-derived route. +- Generated HTML contains seven unique missing targets across eight link occurrences: two malformed DataLad hub paths, two root-style ORCID links, three depiction record links, and `/projects/trr379`. +- `register-depictions.yaml` is only partially repaired at current head. +Its preparation action now comes from the remote Flow repository, but its final step still references the deleted local `./.forgejo/actions/deposit-changes`. +Earlier steps can change/push Annex state before that final failure, so the workflow is not transaction-safe. + +The exact missing targets are in `provenance/upstream-psychoinformatics/missing-targets.tsv`. +The Pages adapter does not hide or reinterpret these content defects; it only keeps root-local paths inside the deployment base path. + +## Static deployment versus metadata refresh + +The upstream deployment workflow is simple and successfully reproduced: checkout, install Git Annex, hydrate Annex, initialize Congo, install Hugo 0.154.5 Extended, and run `hugo --minify`. +It does not invoke the pool, schemas, LinkML, Dump Things, or qri. + +The refresh workflow is materially different: + +1. `dtc get-records` contacts `https://pool.psychoinformatics.de/api`. +2. `qri cache`, exact class filters, inlining, and Jinja rendering create the committed Markdown bundles. +3. `code/pool2graph.py` creates `static/graph.json`. +4. the workflow commits generated content and pushes annex state. + +Its environment is not reproducible from the workflow definition: + +- Flow is referenced as moving `@main`; +- Flow installs qri without an immutable lock; +- qri declares the Dump Things Python client from moving `@master`; +- no Python environment lock or artifact hashes are committed; and +- the full canonical pool input cannot be reconstructed from `psyinf-pool-files-public`, which contains depictions rather than records. + +This explains the apparent paradox: the website is a stable static output once committed, while the process for producing a new output is neither offline nor fully pinned. + +## LinkML bugs, carried patches, and content compensations + +### Why the static trial bypasses the problem + +The static build imports no Python packages and reads no schema. +LinkML's discriminator behavior, Things Schemas' install-time patches, and Dump Things' runtime monkeypatches therefore have no direct effect on rebuilding the committed snapshot. +This is why current upstream can deploy even while the LinkML remediation remains unresolved upstream. + +Those components become relevant when records are validated, converted, loaded into generated models, selected by qri, or re-rendered. +A static success must not be treated as evidence that the metadata stack is reproducible or that discriminator behavior is correct. + +### Current LinkML discriminator state + +The trial identified four related upstream changes, none merged into official LinkML `main` (`c8b9bac95eb62891d8a9e5703a2ce688fdf09ce8` when checked): + +| Concern | Current proposed change | +| --- | --- | +| Preserve an uncompactable URI rather than producing `"None"` | LinkML PR #3839, head `2da67e47...` | +| Dispatch a generated Python subclass from a full URI | LinkML PR #3840, head `bf9903fd...` | +| Permit equivalent CURIE/full-URI values in generated JSON Schema | LinkML PR #3843, head `a285bcfc...` | +| Cross-generator compliance coverage | Draft LinkML PR #3847, head `e0176a27...` | + +These heads are independent, not one installable candidate. +The earlier local composite `793dfc12...` remains valuable evidence against LinkML 1.11.1, but a new pin should build and hash an explicit current composite rather than combine moving PR heads implicitly. + +### Things Schemas file patches + +Things Schemas `d26ea413...` declares only `linkml>=1.11` and an unbounded Dump Things dependency. +Hatch then mutates the installed LinkML tree using `tools/patch_linkml`; there is no lock or derived wheel. + +| Patch | State | Relationship to discriminator work | +| --- | --- | --- | +| `shaclgen_annotations.diff` | Active | SHACL order/path/prefix behavior; unrelated | +| `rdflib_loader_typedesignator.diff` | Active | Populates a designator from RDF `rdf:type`; adjacent but different from generated Python dispatch/JSON Schema | +| `graphqlgen_interface_list.diff` | Active | GraphQL interface syntax; unrelated | +| `linkml_generators_common_ifabsent.diff` | Active | CURIE default generation; URI-adjacent but not discriminator lookup | +| `rdflib_loader_custom_types.diff` | Active | RDF datatype registration; unrelated | +| `linkml_runtime_utils_yamlutils.diff` | Disabled | Its inlined-object fix is already in LinkML 1.11.1 | +| `pythongen_type_reference_order.diff` | Disabled | Reference ordering was reimplemented in LinkML 1.11.1 | +| `jsonschemagen_mixins.diff` | Present but unused | Old mixin proposal; not part of the patch runner | + +The five active patches applied to the earlier LinkML candidate without reject, fuzz, or overlap, and did not modify the candidate's three production files. +That only proves compatibility with the old candidate. +The exercise must be repeated against a current composite. + +### Dump Things runtime monkeypatches + +Dump Things `9f101d97...` imports six monkeypatches whenever its generated-model stack is loaded: + +| Monkeypatch | Assessment | +| --- | --- | +| `compile.py` | Still addresses a meaningful generated module-name issue | +| `enumerations.py` | Useful against 1.11.1, but would overwrite newer LinkML main enum/MRO/`PermissibleValue` behavior | +| `ifabsent_processing.py` | Duplicates the active Things Schemas `ifabsent` patch | +| `pythongen_gen_references.py` | Replaces behavior already fixed in LinkML 1.11.1; retire or version-gate | +| `rdflib_loader.py` | Duplicates the RDF type-designator patch and replaces a whole upstream function, discarding newer namespace, `@base`, enum, and diagnostic improvements | +| `yamlutils.py` | Replaces an already-fixed, more comprehensive 1.11.1 implementation; retire or version-gate | + +This is the highest-risk compatibility boundary. +A dependency resolver can select a newer LinkML version while these full-function replacements silently restore older semantics. + +There are also two different effective patch stacks: + +- Things Schemas development installs LinkML and applies five file patches in its Hatch environment. +- The current CON Pixi site does not install Things Schemas as a package. +It uses pristine locked LinkML 1.11.1 wheels, a vendored schema snapshot, and the six Dump Things runtime monkeypatches. + +Consequently, "the Orinoco LinkML stack" is not one environment. +Schema generation and service/site validation must be recorded and tested separately. + +### Schema identity correction + +The Things Schemas candidate `33604b1a...` is two commits beyond upstream. +It explicitly declares `dlthings:Association`, `dlthings:Attribution`, `dlthings:Generation`, `dlthings:DOI`, and `dlthings:ISSN` in stable and unreleased modules and tests the direct and merged schemas. + +This fixes an identity problem distinct from lexical normalization. +The intended stable identities are Things v2/unreleased URIs; module-derived `things-prov/... +` and `things-publications/... +` URIs remain deliberately invalid. Existing records that use the canonical `dlthings:*` CURIEs do not need a semantic migration after this correction. + +### Why accepted full URIs can still disappear downstream + +LinkML validity alone does not make the current pipeline representation agnostic: + +- `qri list` compares top-level `schema_type` strings exactly; +- `pool2graph.py` dispatches exact `xyzri:*` strings; and +- dataset, instrument, and publication templates test exactly for `dlthings:DOI`. + +qri preserves lexical input. +A full URI can therefore be valid in LinkML and Dump Things yet be omitted from a class-filtered page/graph or rendered as a missing DOI. +The earlier integration trial reproduced those failures. + +The safe publication contract for the current stack is still canonical CURIEs. +A future full-URI policy needs one schema-aware normalization boundary before qri/template/graph dispatch, with adversarial tests proving that wrong or out-of-hierarchy URIs remain rejected. + +### Current CON content compensation + +CON Milestone 1 works around the native-container failure at a deliberately bounded boundary: + +- relationships are stored as PID-valued `dlthings:AttributeSpecification` assertions rather than native Association, Attribution, or Generation containers; +- DOI/ISSN notations are generic `dlthings:Identifier` values rather than typed DOI/ISSN subclasses; +- source roles such as `marcrel:aut` and `marcrel:led` remain in structured migration provenance; and +- `scripts/project_records.py` converts a fixed predicate set into generic site edges while preserving each original source/predicate/target assertion. + +This compensation retains graph and page utility, but it does not claim native qualified-edge or typed-identifier semantics. +It should remain until a pinned LinkML/schema/service/client/qri tuple passes native fixtures end to end. +Then the records can be migrated to native structures and only the compatibility normalization removed; generic route, taxonomy, backlink, and graph generation can remain. + +## Recommended blend with the existing effort + +This branch is a useful baseline, but it should not erase the distinction between static reproduction and canonical-data publication. + +Recommended sequence: + +1. Keep `5b401e0c`, its Annex manifest, Congo, Hugo, and the Pages adapter as a frozen current-upstream smoke/reference layer. +2. Retain `6945272e` as the smaller controlled discriminator comparison. +The current snapshot's 731 extra publications add test cost and review noise without adding discriminator coverage. +3. Rebase or replay the existing discriminator work onto this parent branch in separate commits only after this baseline is accepted. +Keep the LinkML composite, schema identity correction, and downstream normalization tests independent from the site snapshot/gitlink change. +4. Gate/remove obsolete Dump Things monkeypatches before testing against a new LinkML composite. +Record base wheel hashes, patch hashes, and the effective patched tree or derived wheels; a package version lock is insufficient for post-install mutation. +5. Run the five native classes through direct service post, qri cache/list and inlining, template rendering, and graph generation with both canonical CURIE and equivalent full URI fixtures. +Keep invalid modular, unknown, and wrong-hierarchy values as negative controls. +6. Keep CON's canonical YAML and ephemeral validation design. +Use the upstream committed website as a visual/projection reference, not as the source of truth for CON metadata. +7. Before a durable Pages deployment, mirror all annex content, fix or pin the refresh actions/toolchain, and decide whether to repair the 26 dead graph targets and seven missing HTML targets upstream or filter them during projection. + +This gives the requested reassurance: all upstream presentation code and its current generated content can run and can be adapted to GitHub Pages without a persistent metadata service. +It also shows precisely what that success does not reproduce—the canonical pool, a locked refresh toolchain, Annex object custody, and a representation-safe LinkML-to-qri boundary. + +## Deliverables + +- `.github/workflows/upstream-pages-trial.yml`: manual, pinned Pages workflow +- `tools/build_upstream_site.sh`: local upstream checkout/hydrate/build entry point +- `tools/adapt_upstream_pages.py`: artifact-only project-path adapter and audit +- `tests/test_adapt_upstream_pages.py`: focused rewrite, validation, and idempotence tests +- `provenance/upstream-psychoinformatics/baseline.yaml`: exact commits, versions, counts, hashes, and Annex availability +- `provenance/upstream-psychoinformatics/missing-targets.tsv`: exact upstream dead-link evidence +- `provenance/upstream-psychoinformatics/submodule-inventory.tsv`: all parent pins, live upstream heads, workflow roles, and `con` mirror state + +No GitHub Pages site was enabled, no workflow was dispatched, and no production deployment or DNS record was changed. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..ddbed1a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "orinoco-clean-migration-browser-tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "orinoco-clean-migration-browser-tests", + "devDependencies": { + "@playwright/test": "1.62.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2b8022f --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "orinoco-clean-migration-browser-tests", + "private": true, + "type": "module", + "scripts": { + "test:browser": "playwright test" + }, + "devDependencies": { + "@playwright/test": "1.62.1" + } +} diff --git a/pixi.lock b/pixi.lock new file mode 100644 index 0000000..0da7be0 --- /dev/null +++ b/pixi.lock @@ -0,0 +1,2846 @@ +version: 7 +platforms: +- name: linux-64 + virtual-packages: + - __unix=0=0 + - __linux=4.18 + - __glibc=2.28 + - __archspec=0=x86_64 +- name: p1 + subdir: osx-arm64 + virtual-packages: + - __osx=14.0 + - __unix=0=0 + - __archspec=0=m1 +environments: + default: + channels: + - url: https://conda.anaconda.org/conda-forge/ + indexes: + - https://pypi.org/simple + packages: + linux-64: + - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/git-annex-10.20260601-nodep_h1234567_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/hugo-0.154.5-hecca717_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-h54a6638_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_102.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.1-hecca717_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.1.0-ha9f2e26_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.1.0-he0feb66_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-hf4e2dac_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.1.0-h934c35e_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.2-h5347b49_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libuv-1.52.1-h280c20c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.38-h280c20c_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/nodejs-22.23.2-h273caaf_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.3-h35e630c_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd70dff1_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - pypi: ./submodules/dump-things-pyclient + - pypi: ./submodules/dump-things-service + - pypi: ./submodules/query-things + - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/09/05/d129d016f5124adb882816bdaef44bb877e313ceb0a109abcf553f1ac90c/pyjsg-0.12.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/56/89866e9995fdb2c8e8ff1336c4ecd4c86ba0f7e4622ccfacad2c13b2ba7e/chardet-7.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/c2/6604a71269e0c1bd75656d5a001432d16f2cc5b8c057140ec797155c295e/rdflib-7.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/fb/3068f649cc436be915f51b2f5ac0656c83dc9bcc6d4f8940633e295042c0/linkml-1.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/28/7f/cfd9bc4b1f5e424eeea83d0493e43f3b1b02707ce8e50c47945873982bd5/wrapt-2.3.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/e2/70692eba662037cddf93391cbbf98297159f3038612e9b9a8129e16feb7a/sentry_sdk-2.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz + - pypi: https://files.pythonhosted.org/packages/40/ea/66c21d1f5fec82e6218a70b5672870f76878f41bf3b9570235b4e7223118/pyshex-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/42/28/9ec976c16d63cc51c28dfec74b66854048c0b8b6579946e902f91b69e8bf/websockets-17.0.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/42/f7/7a3935abdebd5cf18705a5f0335dd6a3a18bef3baa7cb9edc3b6b9922cc8/typing_inspection-0.4.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/44/bd/cb244695f67f77b0a36200ce1670fc42a6fe2770847e870daab99cc2b177/sphinx_click-6.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/74/d5405b9b3b12e9176dff223576d7090bc161092878f533fd0dc23dd6ae1d/looseversion-1.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/62/3bcf988945065eb9f6ce91d8f5ecdcab59093a539549c3d0d9ec777d75e5/curies-0.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5d/27/a7bc8ecff2c5791c6d202a71bdb93b0c48ac97ff71d08350405b50e96fe0/snapper_fmt-0.8.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5e/1d/d8d5be9e72e518b42f544e196de9c07161b0933143c9d0e4e2e33de60d79/pyshexc-0.10.3.post1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/1d/600b0dd24aa61f03d35293a2e9a4695add1e94c03d8701436fb52d5daf4f/linkml_runtime-1.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/07/a89b539750a159d5101c4eb9fc84e2961f65cefbd5e0b7440b284471c0b0/python_discovery-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6b/b2/d17b2722c636d64b4e77ddc68d8d0625719d39f94021be8719a218af4c0a/backports_zstd-1.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/6d/94/b7123440be1490730cbef0b2e01b9d47c6d4a1b206c87289a1ca9a6cdebb/datalad_core-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/97/a87901aef6b7e7e4a34c6dd6cc17dca8594a592ef9d9dd765fca2b7facf7/rich_click-1.9.8-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6f/c5/7c16e99869e1f422629092cfd23e3b58e461988c3f9c36fd3624bb4142e6/parse-1.22.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/75/45/54bb2d8d4138964a94bef6e9afe48b0be4705ba66ac442ae7d8a8dc4ffef/click_option_group-0.5.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/2a/83d779d2dfb61f101d7b1c10073d18984e37262d8b4f171c99911a952430/virtualenv-21.7.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/86/72/b03ca1560615933f079ba7d291d3532ed95c2a3205911fe71d192654acaa/shexjsg-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a4/d1/9725ec62421dcfe430d874cba9f82c0288baee0c4b7dd8aa8dcca2b0394e/fastapi_pagination-0.15.16-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a9/a8/c6e6db62226df8cd3d950889925d826a0e35ac567cc5df548a872944f1ed/rignore-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c7/d5/68e6e9bca63c0badf67002890a46d3784c958de45b65e1275ec583ca1f06/uvicorn-0.52.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c9/13/c51c203d05c765e29b83006eba89bfec3f41e72407668f8beee482153e9f/datasalad-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz + - pypi: https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/7e/e7ceaf949ecee97ba6003872b3d88bed1b22941742381030d9fc8ea4525e/pystow-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl + p1: + - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/hugo-0.154.5-ha4d9615_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-hc7cc350_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-22.1.8-h55c6f16_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-hcf2aa1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-h1ae2325_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-22.23.2-h35957e4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.3-hd24854e_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-hd1323d7_1_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hd3d0363_3.conda + - pypi: ./submodules/dump-things-pyclient + - pypi: ./submodules/dump-things-service + - pypi: ./submodules/query-things + - pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/09/05/d129d016f5124adb882816bdaef44bb877e313ceb0a109abcf553f1ac90c/pyjsg-0.12.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/0c/76/f7c02efde81ebb9993586f9e435d2fd1191a6f806f640e4eeb8d004493ed/backports_zstd-1.6.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/10/c2/6604a71269e0c1bd75656d5a001432d16f2cc5b8c057140ec797155c295e/rdflib-7.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1a/ff/b8c3ab819c4b90abeb197f24abb373391daa472c066605247d032941aefe/snapper_fmt-0.8.1-py3-none-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/1f/fb/3068f649cc436be915f51b2f5ac0656c83dc9bcc6d4f8940633e295042c0/linkml-1.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/34/e2/70692eba662037cddf93391cbbf98297159f3038612e9b9a8129e16feb7a/sentry_sdk-2.67.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz + - pypi: https://files.pythonhosted.org/packages/3d/cb/78c08cc3bc134ed95ac9da792bc6cd2f28c3c785e706b2fc214d0c9c3848/git_annex-10.20260601-py3-none-macosx_14_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz + - pypi: https://files.pythonhosted.org/packages/3e/6c/1fe281b2e9f8876ee5cd1b02ae891312c6543d60764c2a937dc4a5f28285/rignore-0.8.1-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/40/4c/eb2f52aeeaf30dbd073d315a251a63ae2b8263171ec4428c135140cb0802/virtualenv-21.7.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/40/ea/66c21d1f5fec82e6218a70b5672870f76878f41bf3b9570235b4e7223118/pyshex-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/42/f7/7a3935abdebd5cf18705a5f0335dd6a3a18bef3baa7cb9edc3b6b9922cc8/typing_inspection-0.4.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/44/bd/cb244695f67f77b0a36200ce1670fc42a6fe2770847e870daab99cc2b177/sphinx_click-6.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4e/74/d5405b9b3b12e9176dff223576d7090bc161092878f533fd0dc23dd6ae1d/looseversion-1.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/62/3bcf988945065eb9f6ce91d8f5ecdcab59093a539549c3d0d9ec777d75e5/curies-0.15.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5e/1d/d8d5be9e72e518b42f544e196de9c07161b0933143c9d0e4e2e33de60d79/pyshexc-0.10.3.post1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/1d/600b0dd24aa61f03d35293a2e9a4695add1e94c03d8701436fb52d5daf4f/linkml_runtime-1.11.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/07/a89b539750a159d5101c4eb9fc84e2961f65cefbd5e0b7440b284471c0b0/python_discovery-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/94/b7123440be1490730cbef0b2e01b9d47c6d4a1b206c87289a1ca9a6cdebb/datalad_core-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6d/97/a87901aef6b7e7e4a34c6dd6cc17dca8594a592ef9d9dd765fca2b7facf7/rich_click-1.9.8-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6e/55/51b92daaf6defb57f4dc56bdcce985400f75c6984a03ca5e78ccac717028/wrapt-2.3.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/6f/c5/7c16e99869e1f422629092cfd23e3b58e461988c3f9c36fd3624bb4142e6/parse-1.22.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl + - pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/75/45/54bb2d8d4138964a94bef6e9afe48b0be4705ba66ac442ae7d8a8dc4ffef/click_option_group-0.5.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/86/72/b03ca1560615933f079ba7d291d3532ed95c2a3205911fe71d192654acaa/shexjsg-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a4/d1/9725ec62421dcfe430d874cba9f82c0288baee0c4b7dd8aa8dcca2b0394e/fastapi_pagination-0.15.16-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c4/8e/847935c588455b0d82fa57a5a8ced4c73a928e30f2012639228e566e3283/chardet-7.5.1-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/c7/d5/68e6e9bca63c0badf67002890a46d3784c958de45b65e1275ec583ca1f06/uvicorn-0.52.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c9/13/c51c203d05c765e29b83006eba89bfec3f41e72407668f8beee482153e9f/datasalad-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz + - pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/cc/a6/d5e2a4e48495616440a21eed07558219ca90243ad00b0502586f95bd4833/fastar-0.11.0-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d9/82/123660edc759c225626b3b91952c7625f85c77a8362acbc35a4623120f7d/websockets-17.0.1-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz + - pypi: https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/7e/e7ceaf949ecee97ba6003872b3d88bed1b22941742381030d9fc8ea4525e/pystow-0.9.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl +packages: +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + build_number: 20 + sha256: 1dd3fffd892081df9726d7eb7e0dea6198962ba775bd88842135a4ddb4deb3c9 + md5: a9f577daf3de00bca7c3c76c0ecbd1de + depends: + - __glibc >=2.17,<3.0.a0 + - libgomp >=7.5.0 + constrains: + - openmp_impl <0.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + run_exports: + strong: + - _openmp_mutex >=4.5 + size: 28948 + timestamp: 1770939786096 +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda + sha256: 1a0d382c515ebf55f8ee1f38c8b81bc95af5c2acc42ad53b66bc5df932032f96 + md5: e675fabcf81499adc7edf58124fb1e01 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + run_exports: + weak: + - bzip2 >=1.0.8,<2.0a0 + size: 257808 + timestamp: 1785906269155 +- conda: https://conda.anaconda.org/conda-forge/linux-64/git-annex-10.20260601-nodep_h1234567_0.conda + sha256: 1df1ee65626f94b426e73080554f1c624c98c8f89ede8935fb53e5ede699b967 + md5: b082ab6b8fa11b1eb4630874b8dc8db9 + constrains: + - gnupg >=2.1.1 + license: AGPL-3.0-only + license_family: AGPL + purls: [] + run_exports: {} + size: 38538511 + timestamp: 1781871477793 +- conda: https://conda.anaconda.org/conda-forge/linux-64/hugo-0.154.5-hecca717_0.conda + sha256: d0d56f2985803009f5248a76ba33baf272b41b528fe9208a0d7c2885690c51ab + md5: ea4d50a3273a01ba462643709d71b9cd + depends: + - __glibc >=2.17 + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + license: Apache-2.0 + license_family: APACHE + purls: [] + run_exports: {} + size: 18403168 + timestamp: 1768298518899 +- conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-h54a6638_2.conda + sha256: d7c260b7e1cf22ce04d6ba8a86eabf4e6c50bc96a5c27fe2ecb32298af3e88eb + md5: 4ef4b977bb216a3001a3334696a80850 + depends: + - libstdcxx >=14 + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14455340 + timestamp: 1784916378180 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.46.1-default_hbd61a6d_102.conda + sha256: 27d83f1188cd19bcb7754a078b3fa7f4cfb8527f8eb2fde54dd01fc529d1adec + md5: 449500f2c089da11c40f5c21312e3e07 + depends: + - __glibc >=2.17,<3.0.a0 + - zstd >=1.5.7,<1.6.0a0 + constrains: + - binutils_impl_linux-64 2.46.1 + license: GPL-3.0-only + license_family: GPL + purls: [] + run_exports: {} + size: 745303 + timestamp: 1784214507189 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.8.1-hecca717_1.conda + sha256: 16feffd9ddbbe5b718515d38ee376c685ba95491cd901244e24671d20b952a77 + md5: b24d3c612f71e7aa74158d92106318b2 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + run_exports: {} + size: 77856 + timestamp: 1781203599810 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda + sha256: 31f19b6a88ce40ebc0d5a992c131f57d919f73c0b92cd1617a5bec83f6e961e6 + md5: a360c33a5abe61c07959e449fa1453eb + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - libffi >=3.5.2,<3.6.0a0 + size: 58592 + timestamp: 1769456073053 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-16.1.0-ha9f2e26_1.conda + sha256: d5cb8475131c31680f8fd30512c418f373064e272e452063276a8fb14c9fa42f + md5: 5a7d954665c707c93311657cd779c705 + depends: + - __glibc >=2.17,<3.0.a0 + - _openmp_mutex >=4.5 + constrains: + - libgomp 16.1.0 he0feb66_1 + - libgcc-ng ==16.1.0=*_1 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + run_exports: {} + size: 1057877 + timestamp: 1785375436766 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-16.1.0-he0feb66_1.conda + sha256: 62cb599ad0539d99386515326d9d5e8f51f75a60c69c2131b21df76edf35bd89 + md5: 88f2d91cb1533194c323534253094d23 + depends: + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + run_exports: + strong: + - _openmp_mutex >=4.5 + size: 640415 + timestamp: 1785375373755 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.3-hb03c661_1.conda + sha256: 9787df8c22a59c9a70d3e5a10db9ad663485e75e9ccc3f09bd092cb7b95e0dab + md5: 1390b7c5ac0b1d8e447bc5efa6d3c8c2 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + run_exports: + weak: + - liblzma >=5.8.3,<6.0a0 + size: 112995 + timestamp: 1786348617826 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + sha256: 927fe72b054277cde6cb82597d0fcf6baf127dcbce2e0a9d8925a68f1265eef5 + md5: d864d34357c3b65a4b731f78c0801dc4 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + license: LGPL-2.1-only + license_family: GPL + purls: [] + run_exports: + weak: + - libnsl >=2.0.1,<2.1.0a0 + size: 33731 + timestamp: 1750274110928 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-hf4e2dac_0.conda + sha256: 72023efc207fe681e26b65fc9d668062cf0b4f0eacf3431e6eb099b95c1f2efd + md5: df088a279cd5e6fd2790b4c196434da1 + depends: + - __glibc >=2.17,<3.0.a0 + - icu >=78.3,<79.0a0 + - libgcc >=14 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + run_exports: + weak: + - libsqlite >=3.53.4,<4.0a0 + size: 964200 + timestamp: 1785016112246 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.1.0-h934c35e_1.conda + sha256: 79721dd08aeb0ab9e773f1f9ef41cf4e6c17477e3d72319147619045bce05a09 + md5: aed6cf89adc1e9b846e4367ac538e434 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 16.1.0 ha9f2e26_1 + constrains: + - libstdcxx-ng ==16.1.0=*_1 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + run_exports: {} + size: 6631744 + timestamp: 1785375462643 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.42.2-h5347b49_0.conda + sha256: 9b1bdce27a7e31f7d241aeecff67a1f3101d52a2b1e33ccc2cdf2613072bf81f + md5: 01bb81d12c957de066ea7362007df642 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + run_exports: + weak: + - libuuid >=2.42.2,<3.0a0 + size: 40017 + timestamp: 1781625522462 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuv-1.52.1-h280c20c_1.conda + sha256: 67761d0206f84140047a367eaf9befe03a7e157a29ee25aee0047b40801cc6b6 + md5: 01c4ed87826af55996768af6dbc936b4 + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 420040 + timestamp: 1785914567661 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.38-h280c20c_0.conda + sha256: f7e9292dd219a6435bbb1223da9586c3e70d66d169c5a92f08db3f2127df04e9 + md5: f7a7ff5a6ab331e037abd34f379a631d + depends: + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + license: LGPL-2.1-or-later + purls: [] + run_exports: + weak: + - libxcrypt >=4.4.38 + size: 101957 + timestamp: 1785887123445 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.2-h25fd6f3_3.conda + sha256: eb8a0db0aa570124f7d2a93d7c7f596e3390df5e047818d873baad32985fc736 + md5: 0de0122d9570a8ab637c6b73db268389 + depends: + - __glibc >=2.17,<3.0.a0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + purls: [] + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 63713 + timestamp: 1785362952714 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda + sha256: 5d46557214ed184381dafe835b7c94a474a1c3b307a08a250b1ea4779b44ffb3 + md5: ee6c0cd80a60961a1f48aa3e0b91f986 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + license: X11 AND BSD-3-Clause + purls: [] + run_exports: + weak: + - ncurses >=6.6,<7.0a0 + size: 911196 + timestamp: 1786355078102 +- conda: https://conda.anaconda.org/conda-forge/linux-64/nodejs-22.23.2-h273caaf_0.conda + sha256: 3bbfd6f93d029df231782f5ddfec59d7d013331d41b66108860affa6d97fb992 + md5: 6ec38173d646f6437704816eb9423836 + depends: + - __glibc >=2.28,<3.0.a0 + - libgcc >=14 + - libstdcxx >=14 + - libuv >=1.52.1,<2.0a0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + - openssl >=3.5.7,<4.0a0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - nodejs >=22.23.2,<23.0a0 + size: 24147385 + timestamp: 1785913890438 +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.3-h35e630c_1.conda + sha256: 012096056b97abf1f68c46b7146bd2cbd68c1be762340b4f5dad4fbbe99177bc + md5: c5955c27917ff2234def47f075e71e02 + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=14 + license: Apache-2.0 + license_family: Apache + purls: [] + run_exports: + weak: + - openssl >=3.6.3,<4.0a0 + size: 3182423 + timestamp: 1785913583650 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.12.13-hd63d673_0_cpython.conda + sha256: a44655c1c3e1d43ed8704890a91e12afd68130414ea2c0872e154e5633a13d7e + md5: 7eccb41177e15cc672e1babe9056018e + depends: + - __glibc >=2.17,<3.0.a0 + - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.7.4,<3.0a0 + - libffi >=3.5.2,<3.6.0a0 + - libgcc >=14 + - liblzma >=5.8.2,<6.0a0 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.51.2,<4.0a0 + - libuuid >=2.41.3,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.3.1,<2.0a0 + - ncurses >=6.5,<7.0a0 + - openssl >=3.5.5,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + run_exports: + weak: + - python_abi 3.12.* *_cp312 + noarch: + - python + size: 31608571 + timestamp: 1772730708989 +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + sha256: 12ffde5a6f958e285aa22c191ca01bbd3d6e710aa852e00618fa6ddc59149002 + md5: d7d95fc8287ea7bf33e0e7116d2b95ec + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + run_exports: + weak: + - readline >=8.3,<9.0a0 + size: 345073 + timestamp: 1765813471974 +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_hd70dff1_3.conda + build_number: 103 + sha256: 43624eab22f5f29df7d6ffe914cf442f28fd559b55b290906255492826e636e8 + md5: 48a1049e710857572fc2a832aa394d9f + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libzlib >=1.3.2,<2.0a0 + constrains: + - xorg-libx11 >=1.8.13,<2.0a0 + license: TCL + purls: [] + run_exports: + weak: + - tk >=8.6.13,<8.7.0a0 + size: 3550916 + timestamp: 1784229071544 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + sha256: 68f0206ca6e98fea941e5717cec780ed2873ffabc0e1ed34428c061e2c6268c7 + md5: 4a13eeac0b5c8e5b8ab496e6c4ddd829 + depends: + - __glibc >=2.17,<3.0.a0 + - libzlib >=1.3.1,<2.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + run_exports: + weak: + - zstd >=1.5.7,<1.6.0a0 + size: 601375 + timestamp: 1764777111296 +- conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.7.22-hbd8a1cb_0.conda + sha256: 0a0544cf95f64394fe4959286f5c71f5444ad58feb0602e53becb27448d24da6 + md5: 0f51e2391ade309db462a55611263e9c + depends: + - __unix + license: ISC + purls: [] + run_exports: {} + size: 131780 + timestamp: 1784754889428 +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda + sha256: b928c30ddcb0e3f544c6eade8352737e6e610e263276b90232db6a578ef899d8 + md5: fcb489df604d100968b737f2cb6076c6 + license: LicenseRef-Public-Domain + purls: [] + run_exports: {} + size: 118849 + timestamp: 1784250406640 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h4e30115_10.conda + sha256: 8ec22f0ba25cbfc2e64d70cf29459eccd7ffdf6436f6a6ff15bbfef799f7d4f6 + md5: b50612e7d190b8061ab4e7dc119cf4d5 + depends: + - __osx >=11.0 + license: bzip2-1.0.6 + license_family: BSD + purls: [] + run_exports: + weak: + - bzip2 >=1.0.8,<2.0a0 + size: 124965 + timestamp: 1785906749812 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/hugo-0.154.5-ha4d9615_0.conda + sha256: a87f25e4890b876cc9b6d3d9cd035290dc4edf11a3f84f5093cc06d064bd5fb3 + md5: e922799404b96cc8d9529efd335b4ff7 + depends: + - __osx >=12.3 + - libcxx >=19 + license: Apache-2.0 + license_family: APACHE + purls: [] + run_exports: {} + size: 16271300 + timestamp: 1768299292109 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-hc7cc350_2.conda + sha256: f0b22bc30e4cc29e29ba3234cb38497fe8def2c2aae4b775d42fe5b378a018c9 + md5: 6133ddbb17ba2b50700dd88e9303ce27 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - icu >=78.3,<79.0a0 + size: 14070698 + timestamp: 1784916459058 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-22.1.8-h55c6f16_0.conda + sha256: a2e7abab5add9750fab064c024394de48e49f97631c605ad5db5c8ac3fc769ef + md5: 89f76a2a21a3ec3ec983b5eb237c4113 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + purls: [] + run_exports: {} + size: 569349 + timestamp: 1781670209146 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.8.1-hf6b4638_1.conda + sha256: 5af74261101e3c777399c6294b2b5d290e508153268eb2e9ff99c4d69834612f + md5: a915151d5d3c5bf039f5ccc8402a436f + depends: + - __osx >=11.0 + constrains: + - expat 2.8.1.* + license: MIT + license_family: MIT + purls: [] + run_exports: {} + size: 69362 + timestamp: 1781203631990 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.7.0-hcf2aa1b_0.conda + sha256: 2c6ac9a6cd65af89b2bd448518bb1e13b44a2e48c0d469398e37bcfc0092e832 + md5: 92e8690d170d46d768c32553458c0105 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - libffi >=3.7.0,<3.8.0a0 + size: 43734 + timestamp: 1783521647536 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.3-h8088a28_1.conda + sha256: 23d0630046a3e8b164d8f80f2b74ed2605af2e7050ab9913018056402fae4311 + md5: 8ab10323068b107661a4b9a4af84f3b5 + depends: + - __osx >=11.0 + constrains: + - xz 5.8.3.* + license: 0BSD + purls: [] + run_exports: + weak: + - liblzma >=5.8.3,<6.0a0 + size: 91720 + timestamp: 1786348695846 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.53.4-h1ae2325_0.conda + sha256: 745662565e103f290e9dc4263bbd88285082f8cf699854fe2d5f1e35a4a0d326 + md5: 0e3477c0c3e718dcf2eb74ccc8f68570 + depends: + - __osx >=11.0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: blessing + purls: [] + run_exports: + weak: + - libsqlite >=3.53.4,<4.0a0 + size: 929203 + timestamp: 1785016131414 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libuv-1.52.1-h1a92334_1.conda + sha256: 4f47de9de1990efd998edbbd6793f89c8f02ffde987ae8120b1c006acefd2a04 + md5: de09bd0f175611e94f21b28f8c708e80 + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - libuv >=1.52.1,<2.0a0 + size: 122729 + timestamp: 1785914645797 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.2-h8088a28_3.conda + sha256: a18fa5d5bac452401459f966cf0d872224e8080c4ff93c77e168d43ab42ef9d7 + md5: f39288f0ea63ae962e1a2e4f355a0d75 + depends: + - __osx >=11.0 + constrains: + - zlib 1.3.2 *_3 + license: Zlib + license_family: Other + purls: [] + run_exports: + weak: + - libzlib >=1.3.2,<2.0a0 + size: 47822 + timestamp: 1785277049190 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.6-he64c551_1.conda + sha256: 7024a48c8c0d0114ed4ab53c76bf9275d50e91ba7cea367a9aead638d3c29c68 + md5: 3dfa0d0316dc246cd44937a557de4501 + depends: + - __osx >=11.0 + license: X11 AND BSD-3-Clause + purls: [] + run_exports: + weak: + - ncurses >=6.6,<7.0a0 + size: 804298 + timestamp: 1786355189145 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/nodejs-22.23.2-h35957e4_0.conda + sha256: f147b779805145e67bb0588652b297ed3de90490da693fac05cb3593c03b49c9 + md5: 0ae64df299d12a790bbce918f913c90c + depends: + - __osx >=12.0 + - libcxx >=19 + - openssl >=3.5.7,<4.0a0 + - libuv >=1.52.1,<2.0a0 + - icu >=78.3,<79.0a0 + - libzlib >=1.3.2,<2.0a0 + license: MIT + license_family: MIT + purls: [] + run_exports: + weak: + - nodejs >=22.23.2,<23.0a0 + size: 16415636 + timestamp: 1785913942109 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.3-hd24854e_1.conda + sha256: 66be2283b5b37dcda1332b5e74c1782a8cb14fd2e62e0d38017c2d35bf73c119 + md5: 65d1906712b85d1679263c518d011b5b + depends: + - __osx >=11.0 + - ca-certificates + license: Apache-2.0 + license_family: Apache + purls: [] + run_exports: + weak: + - openssl >=3.6.3,<4.0a0 + size: 3109132 + timestamp: 1785913735357 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.13-hd1323d7_1_cpython.conda + build_number: 1 + sha256: b375287c4fa8737c0a44af917d4c2b2cb9cd79e85d224733cd2b46165e9988b1 + md5: c83039bc99cd4b90204ca5c96f7fddda + depends: + - __osx >=11.0 + - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.8.1,<3.0a0 + - libffi >=3.7.0,<3.8.0a0 + - liblzma >=5.8.3,<6.0a0 + - libsqlite >=3.53.4,<4.0a0 + - libzlib >=1.3.2,<2.0a0 + - ncurses >=6.6,<7.0a0 + - openssl >=3.5.7,<4.0a0 + - readline >=8.3,<9.0a0 + - tk >=8.6.13,<8.7.0a0 + - tzdata + constrains: + - python_abi 3.12.* *_cp312 + license: Python-2.0 + purls: [] + run_exports: + weak: + - python_abi 3.12.* *_cp312 + noarch: + - python + size: 13473493 + timestamp: 1786444752563 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + sha256: a77010528efb4b548ac2a4484eaf7e1c3907f2aec86123ed9c5212ae44502477 + md5: f8381319127120ce51e081dce4865cf4 + depends: + - __osx >=11.0 + - ncurses >=6.5,<7.0a0 + license: GPL-3.0-only + license_family: GPL + purls: [] + run_exports: + weak: + - readline >=8.3,<9.0a0 + size: 313930 + timestamp: 1765813902568 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-hd3d0363_3.conda + sha256: 47186bc7ab8d7e8bee86bbd1a917196f8c21cf63f081fc33cd6d1221af087580 + md5: 8e3cf0e455e6b54519f0b1c72c61780a + depends: + - __osx >=11.0 + - libzlib >=1.3.2,<2.0a0 + license: TCL + purls: [] + run_exports: + weak: + - tk >=8.6.13,<8.7.0a0 + size: 3338712 + timestamp: 1784229090530 +- pypi: ./submodules/dump-things-pyclient + name: dump-things-pyclient + requires_dist: + - click + - jsonpath-ng + - pyyaml + - requests + - rich-click + - dump-things-service>=5.6.1 ; extra == 'ttl' + - dump-things-service>=6.2.2 ; extra == 'tests' + - pytest>=9.0.1 ; extra == 'tests' + requires_python: '>=3.11' +- pypi: ./submodules/dump-things-service + name: dump-things-service + requires_dist: + - aiohttp + - click + - datalad-core + - fastapi-pagination>=0.15.15 + - fastapi[standard]>=0.138.0 + - fsspec + - linkml>=1.10.0 + - pydantic + - pyyaml + - rdflib + - requests + - sqlalchemy + - uvicorn + - myst-parser ; extra == 'docs' + - pytest ; extra == 'docs' + - sphinx ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + requires_python: '>=3.11' +- pypi: ./submodules/query-things + name: query-things + requires_dist: + - click + - click-option-group + - dump-things-pyclient @ git+https://hub.psychoinformatics.de/orinoco/dump-things-pyclient.git@master + - jinja2 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/00/cc/7fbd75d3362e939eb98bcf9bd22f3f7df8c237a85148899ed3d38e5614e5/json_flattener-0.1.9-py3-none-any.whl + name: json-flattener + version: 0.1.9 + sha256: 6b027746f08bf37a75270f30c6690c7149d5f704d8af1740c346a3a1236bc941 + requires_dist: + - click + - pyyaml + requires_python: '>=3.7.0' +- pypi: https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: charset-normalizer + version: 3.4.9 + sha256: 5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl + name: distlib + version: 0.4.3 + sha256: 4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b +- pypi: https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl + name: docutils + version: 0.22.4 + sha256: d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl + name: jsonpath-ng + version: 1.8.0 + sha256: b8dde192f8af58d646fc031fac9c99fe4d00326afc4148f1f043c601a8cfe138 +- pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl + name: h11 + version: 0.16.0 + sha256: 63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl + name: roman-numerals + version: 4.1.0 + sha256: 647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: rpds-py + version: 2026.6.3 + sha256: ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/09/05/d129d016f5124adb882816bdaef44bb877e313ceb0a109abcf553f1ac90c/pyjsg-0.12.4-py3-none-any.whl + name: pyjsg + version: 0.12.4 + sha256: a57ae58bfd7192b32654a0024bc6462fb459d54e837f0b2b5cff0726aad2e557 + requires_dist: + - antlr4-python3-runtime~=4.9.3 + - jsonasobj>=1.2.1 + - requests + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl + name: sphinxcontrib-htmlhelp + version: 2.1.0 + sha256: 166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8 + requires_dist: + - ruff==0.5.5 ; extra == 'lint' + - mypy ; extra == 'lint' + - types-docutils ; extra == 'lint' + - sphinx>=5 ; extra == 'standalone' + - pytest ; extra == 'test' + - html5lib ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl + name: certifi + version: 2026.7.22 + sha256: 62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl + name: python-dotenv + version: 1.2.2 + sha256: 1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a + requires_dist: + - click>=5.0 ; extra == 'cli' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/0c/76/f7c02efde81ebb9993586f9e435d2fd1191a6f806f640e4eeb8d004493ed/backports_zstd-1.6.0-cp312-cp312-macosx_11_0_arm64.whl + name: backports-zstd + version: 1.6.0 + sha256: 1d146926e997d2d3de8212bdcbf4985344a2622ca3bec458d8908000a84fd883 + requires_python: '>=3.10,<3.14' +- pypi: https://files.pythonhosted.org/packages/10/56/89866e9995fdb2c8e8ff1336c4ecd4c86ba0f7e4622ccfacad2c13b2ba7e/chardet-7.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: chardet + version: 7.5.1 + sha256: ecbe0e0a9fff7825fc48650ef297ede49c71a7abc411a0638416207a70bf78c0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl + name: pyparsing + version: 3.3.2 + sha256: 850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d + requires_dist: + - railroad-diagrams ; extra == 'diagrams' + - jinja2 ; extra == 'diagrams' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/10/c2/6604a71269e0c1bd75656d5a001432d16f2cc5b8c057140ec797155c295e/rdflib-7.6.0-py3-none-any.whl + name: rdflib + version: 7.6.0 + sha256: 30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd + requires_dist: + - berkeleydb>=18.1.0,<19.0.0 ; extra == 'berkeleydb' + - html5rdf>=1.2,<2 ; extra == 'html' + - httpx>=0.28.1,<0.29.0 ; extra == 'graphdb' or extra == 'rdf4j' + - isodate>=0.7.2,<1.0.0 ; python_full_version < '3.11' + - lxml>=4.3,<6.0 ; extra == 'lxml' + - networkx>=2,<4 ; extra == 'networkx' + - orjson>=3.9.14,<4 ; extra == 'orjson' + - pyparsing>=2.1.0,<4 + requires_python: '>=3.8.1' +- pypi: https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: propcache + version: 0.5.2 + sha256: 6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl + name: isodate + version: 0.7.2 + sha256: 28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl + name: pydantic-extra-types + version: 2.11.1 + sha256: 1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1 + requires_dist: + - pydantic>=2.5.2 + - typing-extensions + - cron-converter>=1.2.2 ; extra == 'all' + - pendulum>=3.0.0,<4.0.0 ; extra == 'all' + - phonenumbers>=8,<10 ; extra == 'all' + - pycountry>=23 ; extra == 'all' + - pymongo>=4.0.0,<5.0.0 ; extra == 'all' + - python-ulid>=1,<2 ; python_full_version < '3.9' and extra == 'all' + - python-ulid>=1,<4 ; python_full_version >= '3.9' and extra == 'all' + - pytz>=2024.1 ; extra == 'all' + - semver>=3.0.2 ; extra == 'all' + - semver~=3.0.2 ; extra == 'all' + - tzdata>=2024.1 ; extra == 'all' + - uuid-utils>=0.6.0 ; python_full_version < '3.14' and extra == 'all' + - cron-converter>=1.2.2 ; extra == 'cron' + - pendulum>=3.0.0,<4.0.0 ; extra == 'pendulum' + - phonenumbers>=8,<10 ; extra == 'phonenumbers' + - pycountry>=23 ; extra == 'pycountry' + - python-ulid>=1,<2 ; python_full_version < '3.9' and extra == 'python-ulid' + - python-ulid>=1,<4 ; python_full_version >= '3.9' and extra == 'python-ulid' + - semver>=3.0.2 ; extra == 'semver' + - uuid-utils>=0.6.0 ; python_full_version < '3.14' and extra == 'uuid-utils' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl + name: pydantic-core + version: 2.46.4 + sha256: 962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/1a/77/48ce09fce2836856588beb84f434c1f8812d1428326efd993b619d49d949/sparqlslurper-0.5.1-py3-none-any.whl + name: sparqlslurper + version: 0.5.1 + sha256: ae49b2d8ce3dd38df7a40465b228ad5d33fb7e11b3f248d195f9cadfc9cfff87 + requires_dist: + - rdflib-shim + - rdflib>=5.0.0 + - sparqlwrapper>=1.8.2 + requires_python: '>=3.7.4' +- pypi: https://files.pythonhosted.org/packages/1a/ff/b8c3ab819c4b90abeb197f24abb373391daa472c066605247d032941aefe/snapper_fmt-0.8.1-py3-none-macosx_11_0_arm64.whl + name: snapper-fmt + version: 0.8.1 + sha256: d59fb92eab4188a6d1336ab379de24a898c78042adc0b239fcf46ca22112b7a1 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl + name: idna + version: '3.18' + sha256: 7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 + requires_dist: + - ruff>=0.6.2 ; extra == 'all' + - mypy>=1.11.2 ; extra == 'all' + - pytest>=8.3.2 ; extra == 'all' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/1f/fb/3068f649cc436be915f51b2f5ac0656c83dc9bcc6d4f8940633e295042c0/linkml-1.11.1-py3-none-any.whl + name: linkml + version: 1.11.1 + sha256: d1bbb97a8b1ea4a99b145007875733a5e5e89b3acfe3e9d1e369fa4a582990ed + requires_dist: + - antlr4-python3-runtime>=4.9.0,<4.10 + - click>=8.2 + - graphviz>=0.10.1 + - hbreader + - isodate>=0.6.0 + - jinja2>=3.1.0 + - jsonasobj2>=1.0.3,<2.0.0 + - jsonschema[format]>=4.0.0 + - linkml-runtime>=1.10.0,<2.0.0 + - openpyxl + - parse + - prefixcommons>=0.1.7 + - prefixmaps>=0.2.2 + - pydantic>=2.0.0,<3.0.0 + - pyjsg>=0.11.6 + - pyshex>=0.7.20 + - pyshexc>=0.8.3 + - python-dateutil + - pyyaml + - rdflib>=6.0.0 + - requests>=2.22 + - sphinx-click>=6.0.0 + - sqlalchemy>=1.4.31 + - typing-extensions>=4.6.0 ; python_full_version < '3.12' + - watchdog>=0.9.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: watchfiles + version: 1.2.0 + sha256: e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5 + requires_dist: + - anyio>=3.0.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: fastar + version: 0.11.0 + sha256: ef5a6071121e05d8287fc75bccb054bcbac8bb0501200a0c0a8feeace5303ea4 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl + name: pytest + version: 9.1.1 + sha256: 37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + requires_dist: + - colorama>=0.4 ; sys_platform == 'win32' + - exceptiongroup>=1 ; python_full_version < '3.11' + - iniconfig>=1.0.1 + - packaging>=22 + - pluggy>=1.5,<2 + - pygments>=2.7.2 + - tomli>=1 ; python_full_version < '3.11' + - argcomplete ; extra == 'dev' + - attrs>=19.2 ; extra == 'dev' + - hypothesis>=3.56 ; extra == 'dev' + - mock ; extra == 'dev' + - requests ; extra == 'dev' + - setuptools ; extra == 'dev' + - xmlschema ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl + name: sphinxcontrib-qthelp + version: 2.0.0 + sha256: b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb + requires_dist: + - ruff==0.5.5 ; extra == 'lint' + - mypy ; extra == 'lint' + - types-docutils ; extra == 'lint' + - sphinx>=5 ; extra == 'standalone' + - pytest ; extra == 'test' + - defusedxml>=0.7.1 ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/28/7f/cfd9bc4b1f5e424eeea83d0493e43f3b1b02707ce8e50c47945873982bd5/wrapt-2.3.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + name: wrapt + version: 2.3.0 + sha256: 5d221a6e6ddd302b8397433184e96b59f259f50024b854db1c411a881586b6b8 + requires_dist: + - pytest ; extra == 'dev' + - setuptools ; extra == 'dev' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl + name: httpx + version: 0.28.1 + sha256: d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + requires_dist: + - anyio + - certifi + - httpcore==1.* + - idna + - brotli ; platform_python_implementation == 'CPython' and extra == 'brotli' + - brotlicffi ; platform_python_implementation != 'CPython' and extra == 'brotli' + - click==8.* ; extra == 'cli' + - pygments==2.* ; extra == 'cli' + - rich>=10,<14 ; extra == 'cli' + - h2>=3,<5 ; extra == 'http2' + - socksio==1.* ; extra == 'socks' + - zstandard>=0.18.0 ; extra == 'zstd' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/2b/7c/7ab9f9aadc5944fdd06612484ed7918fe376ad871a5f50404dc1536e0194/sqlalchemy-2.0.51-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: sqlalchemy + version: 2.0.51 + sha256: 1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9 + requires_dist: + - importlib-metadata ; python_full_version < '3.8' + - greenlet>=1 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64' + - typing-extensions>=4.6.0 + - greenlet>=1 ; extra == 'asyncio' + - mypy>=0.910 ; extra == 'mypy' + - pyodbc ; extra == 'mssql' + - pymssql ; extra == 'mssql-pymssql' + - pyodbc ; extra == 'mssql-pyodbc' + - mysqlclient>=1.4.0 ; extra == 'mysql' + - mysql-connector-python ; extra == 'mysql-connector' + - mariadb>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10 ; extra == 'mariadb-connector' + - cx-oracle>=8 ; extra == 'oracle' + - oracledb>=1.0.1 ; extra == 'oracle-oracledb' + - psycopg2>=2.7 ; extra == 'postgresql' + - pg8000>=1.29.1 ; extra == 'postgresql-pg8000' + - greenlet>=1 ; extra == 'postgresql-asyncpg' + - asyncpg ; extra == 'postgresql-asyncpg' + - psycopg2-binary ; extra == 'postgresql-psycopg2binary' + - psycopg2cffi ; extra == 'postgresql-psycopg2cffi' + - psycopg>=3.0.7 ; extra == 'postgresql-psycopg' + - psycopg[binary]>=3.0.7 ; extra == 'postgresql-psycopgbinary' + - pymysql ; extra == 'pymysql' + - greenlet>=1 ; extra == 'aiomysql' + - aiomysql>=0.2.0 ; extra == 'aiomysql' + - greenlet>=1 ; extra == 'aioodbc' + - aioodbc ; extra == 'aioodbc' + - greenlet>=1 ; extra == 'asyncmy' + - asyncmy>=0.2.3,!=0.2.4,!=0.2.6 ; extra == 'asyncmy' + - greenlet>=1 ; extra == 'aiosqlite' + - aiosqlite ; extra == 'aiosqlite' + - typing-extensions!=3.10.0.1 ; extra == 'aiosqlite' + - sqlcipher3-binary ; extra == 'sqlcipher' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl + name: frozenlist + version: 1.8.0 + sha256: f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl + name: referencing + version: 0.37.0 + sha256: 381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 + requires_dist: + - attrs>=22.2.0 + - rpds-py>=0.7.0 + - typing-extensions>=4.4.0 ; python_full_version < '3.13' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl + name: propcache + version: 0.5.2 + sha256: e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl + name: aiohttp + version: 3.14.3 + sha256: 617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee + requires_dist: + - aiohappyeyeballs>=2.5.0 + - aiosignal>=1.4.0 + - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' + - attrs>=17.3.0 + - frozenlist>=1.1.1 + - multidict>=4.5,<7.0 + - propcache>=0.2.0 + - typing-extensions>=4.4 ; python_full_version < '3.13' + - yarl>=1.17.0,<2.0 + - aiodns>=3.3.0 ; sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotli>=1.2 ; platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' + - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl + name: pydantic-settings + version: 2.15.0 + sha256: 0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42 + requires_dist: + - pydantic>=2.7.0 + - python-dotenv>=0.21.0 + - typing-inspection>=0.4.0 + - boto3>=1.35.0 ; extra == 'aws-secrets-manager' + - azure-identity>=1.16.0 ; extra == 'azure-key-vault' + - azure-keyvault-secrets>=4.8.0 ; extra == 'azure-key-vault' + - google-cloud-secret-manager>=2.23.1 ; extra == 'gcp-secret-manager' + - tomli>=2.0.1 ; extra == 'toml' + - pyyaml>=6.0.1 ; extra == 'yaml' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/31/89/176e3db96e31e795d7dfd91dd67749d3d1f0316bb30c6931a6140e1a0477/SPARQLWrapper-2.0.0-py3-none-any.whl + name: sparqlwrapper + version: 2.0.0 + sha256: c99a7204fff676ee28e6acef327dc1ff8451c6f7217dcd8d49e8872f324a8a20 + requires_dist: + - rdflib>=6.1.1 + - setuptools>=3.7.1 ; extra == 'dev' + - mypy>=0.931 ; extra == 'dev' + - pandas>=1.3.5 ; extra == 'dev' + - pandas-stubs>=1.2.0.48 ; extra == 'dev' + - sphinx<5 ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + - keepalive>=0.5 ; extra == 'keepalive' + - pandas>=1.3.5 ; extra == 'pandas' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/31/e8/715b09df3dab02b07809d812042dc47a46236b5603d9d3a2572dbd1d8a97/prefixcommons-0.1.12-py3-none-any.whl + name: prefixcommons + version: 0.1.12 + sha256: 16dbc0a1f775e003c724f19a694fcfa3174608f5c8b0e893d494cf8098ac7f8b + requires_dist: + - pyyaml>=6.0,<7.0 + - click>=8.1.3,<9.0.0 + - pytest-logging>=2015.11.4,<2016.0.0 + - requests>=2.28.1,<3.0.0 + requires_python: '>=3.7,<4.0' +- pypi: https://files.pythonhosted.org/packages/34/e2/70692eba662037cddf93391cbbf98297159f3038612e9b9a8129e16feb7a/sentry_sdk-2.67.1-py3-none-any.whl + name: sentry-sdk + version: 2.67.1 + sha256: a66bfbce1cd8a93c51c369d642ad85b46253ea7a6f7938141315b83e2823cda5 + requires_dist: + - urllib3>=1.26.11 + - certifi + - aiohttp>=3.5 ; extra == 'aiohttp' + - anthropic>=0.16 ; extra == 'anthropic' + - arq>=0.23 ; extra == 'arq' + - asyncpg>=0.23 ; extra == 'asyncpg' + - apache-beam>=2.12 ; extra == 'beam' + - bottle>=0.12.13 ; extra == 'bottle' + - celery>=3 ; extra == 'celery' + - celery-redbeat>=2 ; extra == 'celery-redbeat' + - chalice>=1.16.0 ; extra == 'chalice' + - clickhouse-driver>=0.2.0 ; extra == 'clickhouse-driver' + - django>=1.8 ; extra == 'django' + - falcon>=1.4 ; extra == 'falcon' + - fastapi>=0.79.0 ; extra == 'fastapi' + - flask>=0.11 ; extra == 'flask' + - blinker>=1.1 ; extra == 'flask' + - markupsafe ; extra == 'flask' + - grpcio>=1.21.1 ; extra == 'grpcio' + - protobuf>=3.8.0 ; extra == 'grpcio' + - httpcore[http2]==1.* ; extra == 'http2' + - httpcore[asyncio]==1.* ; extra == 'asyncio' + - httpx>=0.16.0 ; extra == 'httpx' + - huey>=2 ; extra == 'huey' + - huggingface-hub>=0.22 ; extra == 'huggingface-hub' + - langchain>=0.0.210 ; extra == 'langchain' + - langgraph>=0.6.6 ; extra == 'langgraph' + - launchdarkly-server-sdk>=9.8.0 ; extra == 'launchdarkly' + - litellm>=1.77.5,!=1.82.7,!=1.82.8 ; extra == 'litellm' + - litestar>=2.0.0 ; extra == 'litestar' + - loguru>=0.5 ; extra == 'loguru' + - mcp>=1.15.0 ; extra == 'mcp' + - openai>=1.0.0 ; extra == 'openai' + - tiktoken>=0.3.0 ; extra == 'openai' + - openfeature-sdk>=0.7.1 ; extra == 'openfeature' + - opentelemetry-distro>=0.35b0 ; extra == 'opentelemetry' + - opentelemetry-distro ; extra == 'opentelemetry-experimental' + - opentelemetry-distro[otlp]>=0.35b0 ; extra == 'opentelemetry-otlp' + - pure-eval ; extra == 'pure-eval' + - executing ; extra == 'pure-eval' + - asttokens ; extra == 'pure-eval' + - pydantic-ai>=1.0.0 ; extra == 'pydantic-ai' + - pymongo>=3.1 ; extra == 'pymongo' + - pyspark>=2.4.4 ; extra == 'pyspark' + - quart>=0.16.1 ; extra == 'quart' + - blinker>=1.1 ; extra == 'quart' + - rq>=0.6 ; extra == 'rq' + - sanic>=0.8 ; extra == 'sanic' + - sqlalchemy>=1.2 ; extra == 'sqlalchemy' + - starlette>=0.19.1 ; extra == 'starlette' + - starlite>=1.48 ; extra == 'starlite' + - statsig>=0.55.3 ; extra == 'statsig' + - tornado>=6 ; extra == 'tornado' + - unleashclient>=6.0.1 ; extra == 'unleash' + - google-genai>=1.29.0 ; extra == 'google-genai' + requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl + name: sphinxcontrib-devhelp + version: 2.0.0 + sha256: aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2 + requires_dist: + - ruff==0.5.5 ; extra == 'lint' + - mypy ; extra == 'lint' + - types-docutils ; extra == 'lint' + - sphinx>=5 ; extra == 'standalone' + - pytest ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz + name: sqlalchemy + version: 2.0.52 + sha256: 5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97 + requires_dist: + - importlib-metadata ; python_full_version < '3.8' + - greenlet>=1 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64' + - typing-extensions>=4.6.0 + - greenlet>=1 ; extra == 'asyncio' + - mypy>=0.910 ; extra == 'mypy' + - pyodbc ; extra == 'mssql' + - pymssql ; extra == 'mssql-pymssql' + - pyodbc ; extra == 'mssql-pyodbc' + - mysqlclient>=1.4.0 ; extra == 'mysql' + - mysql-connector-python ; extra == 'mysql-connector' + - mariadb>=1.0.1,!=1.1.2,!=1.1.5,!=1.1.10 ; extra == 'mariadb-connector' + - cx-oracle>=8 ; extra == 'oracle' + - oracledb>=1.0.1 ; extra == 'oracle-oracledb' + - psycopg2>=2.7 ; extra == 'postgresql' + - pg8000>=1.29.1 ; extra == 'postgresql-pg8000' + - greenlet>=1 ; extra == 'postgresql-asyncpg' + - asyncpg ; extra == 'postgresql-asyncpg' + - psycopg2-binary ; extra == 'postgresql-psycopg2binary' + - psycopg2cffi ; extra == 'postgresql-psycopg2cffi' + - psycopg>=3.0.7 ; extra == 'postgresql-psycopg' + - psycopg[binary]>=3.0.7 ; extra == 'postgresql-psycopgbinary' + - pymysql ; extra == 'pymysql' + - greenlet>=1 ; extra == 'aiomysql' + - aiomysql>=0.2.0 ; extra == 'aiomysql' + - greenlet>=1 ; extra == 'aioodbc' + - aioodbc ; extra == 'aioodbc' + - greenlet>=1 ; extra == 'asyncmy' + - asyncmy>=0.2.12 ; extra == 'asyncmy' + - greenlet>=1 ; extra == 'aiosqlite' + - aiosqlite ; extra == 'aiosqlite' + - typing-extensions!=3.10.0.1 ; extra == 'aiosqlite' + - sqlcipher3-binary ; extra == 'sqlcipher' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: markupsafe + version: 3.0.3 + sha256: d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/3d/cb/78c08cc3bc134ed95ac9da792bc6cd2f28c3c785e706b2fc214d0c9c3848/git_annex-10.20260601-py3-none-macosx_14_0_arm64.whl + name: git-annex + version: '10.20260601' + sha256: 6b6cc79b450701baabe88f0ac741896f8397b066b0d650c50527c3340c22041e + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl + name: uvloop + version: 0.22.1 + sha256: fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42 + requires_dist: + - aiohttp>=3.10.5 ; extra == 'test' + - flake8~=6.1 ; extra == 'test' + - psutil ; extra == 'test' + - pycodestyle~=2.11.0 ; extra == 'test' + - pyopenssl~=25.3.0 ; extra == 'test' + - mypy>=0.800 ; extra == 'test' + - setuptools>=60 ; extra == 'dev' + - cython~=3.0 ; extra == 'dev' + - sphinx~=4.1.2 ; extra == 'docs' + - sphinxcontrib-asyncio~=0.3.0 ; extra == 'docs' + - sphinx-rtd-theme~=0.5.2 ; extra == 'docs' + requires_python: '>=3.8.1' +- pypi: https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl + name: annotated-doc + version: 0.0.5 + sha256: 117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz + name: antlr4-python3-runtime + version: 4.9.3 + sha256: f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b + requires_dist: + - typing ; python_full_version < '3.5' +- pypi: https://files.pythonhosted.org/packages/3e/6c/1fe281b2e9f8876ee5cd1b02ae891312c6543d60764c2a937dc4a5f28285/rignore-0.8.1-cp312-cp312-macosx_11_0_arm64.whl + name: rignore + version: 0.8.1 + sha256: b9e7ba47a5bb25ad45d39983047ecfbcba01ecda0145458c548cd3f390b73bb7 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/40/4c/eb2f52aeeaf30dbd073d315a251a63ae2b8263171ec4428c135140cb0802/virtualenv-21.7.4-py3-none-any.whl + name: virtualenv + version: 21.7.4 + sha256: 376ec93cd6aab3044fa395d7db226db38043b7b5748948044b2a87168525e843 + requires_dist: + - distlib>=0.3.7,<1 + - filelock>=3.24.2,<4 ; python_full_version >= '3.10' + - filelock>=3.16.1,<=3.19.1 ; python_full_version < '3.10' + - platformdirs>=3.9.1,<5 + - python-discovery>=1.4.2 + - typing-extensions>=4.13.2 ; python_full_version < '3.11' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/40/ea/66c21d1f5fec82e6218a70b5672870f76878f41bf3b9570235b4e7223118/pyshex-0.9.0-py3-none-any.whl + name: pyshex + version: 0.9.0 + sha256: d81344deed686b7c169f23156221ae281225e2ba02b14fe9810335afdefffa9d + requires_dist: + - cfgraph>=0.2.1 + - chardet + - pyshexc>=0.10.3 + - rdflib-shim + - requests>=2.22.0 + - shexjsg>=0.9.0 + - sparqlslurper>=0.5.1 + - sparqlwrapper>=1.8.5 + - urllib3 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl + name: jsonschema-specifications + version: 2025.9.1 + sha256: 98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe + requires_dist: + - referencing>=0.31.0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/42/28/9ec976c16d63cc51c28dfec74b66854048c0b8b6579946e902f91b69e8bf/websockets-17.0.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + name: websockets + version: 17.0.1 + sha256: f47b0815af3948ec6a440b3afa02f05b18cc0939549e91b5c677b5d9c2c8472a + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/42/f7/7a3935abdebd5cf18705a5f0335dd6a3a18bef3baa7cb9edc3b6b9922cc8/typing_inspection-0.4.3-py3-none-any.whl + name: typing-inspection + version: 0.4.3 + sha256: 5f42b23858a91e0b4ef521f5418f03a0da3c9216fd2995ef5e73463100e676cd + requires_dist: + - typing-extensions>=4.15.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl + name: typer + version: 0.27.1 + sha256: 53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56 + requires_dist: + - shellingham>=1.3.0 + - rich>=13.8.0 + - annotated-doc>=0.0.2 + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/44/bd/cb244695f67f77b0a36200ce1670fc42a6fe2770847e870daab99cc2b177/sphinx_click-6.2.0-py3-none-any.whl + name: sphinx-click + version: 6.2.0 + sha256: 1fb1851cb4f2c286d43cbcd57f55db6ef5a8d208bfc3370f19adde232e5803d7 + requires_dist: + - sphinx>=4.0 + - click>=8.0 + - docutils + - reno ; extra == 'docs' + - pytest ; extra == 'test' + - pytest-cov ; extra == 'test' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl + name: typing-extensions + version: 4.16.0 + sha256: 481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl + name: platformdirs + version: 4.11.2 + sha256: 7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl + name: snowballstemmer + version: 3.1.1 + sha256: 7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752 + requires_python: '>=3.3' +- pypi: https://files.pythonhosted.org/packages/4e/74/d5405b9b3b12e9176dff223576d7090bc161092878f533fd0dc23dd6ae1d/looseversion-1.3.0-py2.py3-none-any.whl + name: looseversion + version: 1.3.0 + sha256: 781ef477b45946fc03dd4c84ea87734b21137ecda0e1e122bcb3c8d16d2a56e0 +- pypi: https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl + name: sphinxcontrib-serializinghtml + version: 2.0.0 + sha256: 6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 + requires_dist: + - ruff==0.5.5 ; extra == 'lint' + - mypy ; extra == 'lint' + - types-docutils ; extra == 'lint' + - sphinx>=5 ; extra == 'standalone' + - pytest ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: aiohttp + version: 3.14.3 + sha256: 543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 + requires_dist: + - aiohappyeyeballs>=2.5.0 + - aiosignal>=1.4.0 + - async-timeout>=4.0,<6.0 ; python_full_version < '3.11' + - attrs>=17.3.0 + - frozenlist>=1.1.1 + - multidict>=4.5,<7.0 + - propcache>=0.2.0 + - typing-extensions>=4.4 ; python_full_version < '3.13' + - yarl>=1.17.0,<2.0 + - aiodns>=3.3.0 ; sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotli>=1.2 ; platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + - brotlicffi>=1.2 ; platform_python_implementation != 'CPython' and extra == 'speedups' + - backports-zstd ; python_full_version < '3.14' and platform_python_implementation == 'CPython' and sys_platform != 'android' and sys_platform != 'ios' and extra == 'speedups' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl + name: pluggy + version: 1.6.0 + sha256: e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + requires_dist: + - pre-commit ; extra == 'dev' + - tox ; extra == 'dev' + - pytest ; extra == 'testing' + - pytest-benchmark ; extra == 'testing' + - coverage ; extra == 'testing' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/55/62/3bcf988945065eb9f6ce91d8f5ecdcab59093a539549c3d0d9ec777d75e5/curies-0.15.0-py3-none-any.whl + name: curies + version: 0.15.0 + sha256: cea20aa3f9911aeb2c955ad03112dfe1df658a8c8b260336bb84595df6838001 + requires_dist: + - pystow>=0.9.0 + - pydantic>=2.0 + - fastapi ; extra == 'fastapi' + - python-multipart ; extra == 'fastapi' + - httpx ; extra == 'fastapi' + - defusedxml ; extra == 'fastapi' + - uvicorn ; extra == 'fastapi' + - flask ; extra == 'flask' + - defusedxml ; extra == 'flask' + - pandas ; extra == 'pandas' + - rdflib ; extra == 'rdflib' + - sqlalchemy ; extra == 'sqlalchemy' + - sqlmodel ; extra == 'sqlmodel' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl + name: watchdog + version: 6.0.0 + sha256: 6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0 + requires_dist: + - pyyaml>=3.10 ; extra == 'watchmedo' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl + name: pre-commit + version: 4.3.0 + sha256: 2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8 + requires_dist: + - cfgv>=2.0.0 + - identify>=1.0.0 + - nodeenv>=0.11.1 + - pyyaml>=5.1 + - virtualenv>=20.10.0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5d/27/a7bc8ecff2c5791c6d202a71bdb93b0c48ac97ff71d08350405b50e96fe0/snapper_fmt-0.8.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: snapper-fmt + version: 0.8.1 + sha256: 8ba795b15bc9ad7e11a337898f9567a9654f6a2655e73518876776b38b572dc1 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl + name: sphinxcontrib-applehelp + version: 2.0.0 + sha256: 4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5 + requires_dist: + - ruff==0.5.5 ; extra == 'lint' + - mypy ; extra == 'lint' + - types-docutils ; extra == 'lint' + - sphinx>=5 ; extra == 'standalone' + - pytest ; extra == 'test' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5e/1d/d8d5be9e72e518b42f544e196de9c07161b0933143c9d0e4e2e33de60d79/pyshexc-0.10.3.post1-py3-none-any.whl + name: pyshexc + version: 0.10.3.post1 + sha256: 5d247f2822ef9864152545935d93a07dce66640608ea9414c96f69da7fe7a168 + requires_dist: + - antlr4-python3-runtime~=4.9.3 + - chardet>=7.4.1 + - jsonasobj>=1.2.1 + - pyjsg>=0.11.10 + - rdflib-shim>=1.0.3 + - shexjsg>=0.8.1 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl + name: imagesize + version: 2.0.0 + sha256: 5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96 + requires_python: '>=3.10,<3.15' +- pypi: https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: uvloop + version: 0.22.1 + sha256: 7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4 + requires_dist: + - aiohttp>=3.10.5 ; extra == 'test' + - flake8~=6.1 ; extra == 'test' + - psutil ; extra == 'test' + - pycodestyle~=2.11.0 ; extra == 'test' + - pyopenssl~=25.3.0 ; extra == 'test' + - mypy>=0.800 ; extra == 'test' + - setuptools>=60 ; extra == 'dev' + - cython~=3.0 ; extra == 'dev' + - sphinx~=4.1.2 ; extra == 'docs' + - sphinxcontrib-asyncio~=0.3.0 ; extra == 'docs' + - sphinx-rtd-theme~=0.5.2 ; extra == 'docs' + requires_python: '>=3.8.1' +- pypi: https://files.pythonhosted.org/packages/5f/8f/6af2ea19065c91d8b0ea3516fdfc8c0d349f407e8e9fbf4e5a17360de8ad/watchfiles-1.2.0-cp312-cp312-macosx_11_0_arm64.whl + name: watchfiles + version: 1.2.0 + sha256: 2d95ddc1eb6914154253d239089900813f6a767e174b8e6a50e7fdacb7e4236c + requires_dist: + - anyio>=3.0.0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.46.4 + sha256: 926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/5f/97/d8a785d2c7131c731c90cb0e65af9400081af4380bea4ec04868dc21aa92/rdflib_shim-1.0.3-py3-none-any.whl + name: rdflib-shim + version: 1.0.3 + sha256: 7a853e7750ef1e9bf4e35dea27d54e02d4ed087de5a9e0c329c4a6d82d647081 + requires_dist: + - rdflib>=5.0.0 + - rdflib-jsonld==0.6.1 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl + name: jinja2 + version: 3.1.6 + sha256: 85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + requires_dist: + - markupsafe>=2.0 + - babel>=2.7 ; extra == 'i18n' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/63/1d/600b0dd24aa61f03d35293a2e9a4695add1e94c03d8701436fb52d5daf4f/linkml_runtime-1.11.1-py3-none-any.whl + name: linkml-runtime + version: 1.11.1 + sha256: b22c77d8fd920d0f4f43a6ece31393dc0b28bb47790f3e1c114210318c36b3da + requires_dist: + - click>=8.2 + - curies>=0.5.4 + - deprecated + - hbreader + - isodate>=0.7.2,<1.0.0 ; python_full_version < '3.11' + - json-flattener>=0.1.9 + - jsonasobj2==1.*,>=1.0.0,>=1.0.4 + - jsonschema>=3.2.0 + - prefixcommons>=0.1.12 + - prefixmaps>=0.1.4 + - pydantic>=1.10.2,<3.0.0 + - pyyaml + - rdflib>=6.0.0 + - requests + - coverage ; extra == 'dev' + - requests-cache ; extra == 'dev' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl + name: packaging + version: '26.3' + sha256: d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl + name: attrs + version: 26.1.0 + sha256: c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl + name: rfc3987 + version: 1.3.8 + sha256: 10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53 +- pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl + name: jsonschema + version: 4.26.0 + sha256: d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce + requires_dist: + - attrs>=22.2.0 + - jsonschema-specifications>=2023.3.6 + - referencing>=0.28.4 + - rpds-py>=0.25.0 + - fqdn ; extra == 'format' + - idna ; extra == 'format' + - isoduration ; extra == 'format' + - jsonpointer>1.13 ; extra == 'format' + - rfc3339-validator ; extra == 'format' + - rfc3987 ; extra == 'format' + - uri-template ; extra == 'format' + - webcolors>=1.11 ; extra == 'format' + - fqdn ; extra == 'format-nongpl' + - idna ; extra == 'format-nongpl' + - isoduration ; extra == 'format-nongpl' + - jsonpointer>1.13 ; extra == 'format-nongpl' + - rfc3339-validator ; extra == 'format-nongpl' + - rfc3986-validator>0.1.0 ; extra == 'format-nongpl' + - rfc3987-syntax>=1.1.0 ; extra == 'format-nongpl' + - uri-template ; extra == 'format-nongpl' + - webcolors>=24.6.0 ; extra == 'format-nongpl' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/6a/07/a89b539750a159d5101c4eb9fc84e2961f65cefbd5e0b7440b284471c0b0/python_discovery-1.5.1-py3-none-any.whl + name: python-discovery + version: 1.5.1 + sha256: ac07f44cade589d954e9d6a1e1468539fdddd2cf676beb51da73e0f156b7c932 + requires_dist: + - filelock>=3.15.4 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + name: frozenlist + version: 1.8.0 + sha256: 494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/6b/b2/d17b2722c636d64b4e77ddc68d8d0625719d39f94021be8719a218af4c0a/backports_zstd-1.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: backports-zstd + version: 1.6.0 + sha256: 1a99710fbb225d459d66def4dc2bb2cd4a9a0bdc8b799fc0621cfdd863be9c93 + requires_python: '>=3.10,<3.14' +- pypi: https://files.pythonhosted.org/packages/6d/94/b7123440be1490730cbef0b2e01b9d47c6d4a1b206c87289a1ca9a6cdebb/datalad_core-0.3.0-py3-none-any.whl + name: datalad-core + version: 0.3.0 + sha256: ddf377c1e568314336221782c5a59273bb0f91f8cf85b24542c62750622e807e + requires_dist: + - datasalad>=0.7.0 + - looseversion + - more-itertools + - git-annex>=10.20250721 ; extra == 'annex' + - pytest ; extra == 'docs' + - sphinx ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/6d/97/a87901aef6b7e7e4a34c6dd6cc17dca8594a592ef9d9dd765fca2b7facf7/rich_click-1.9.8-py3-none-any.whl + name: rich-click + version: 1.9.8 + sha256: 12873865396e6927835d4eabb1cc3996edcd65b7ac9b2391a29eca4f335a2f93 + requires_dist: + - click>=8 + - colorama ; sys_platform == 'win32' + - rich>=12 + - typing-extensions>=4 ; python_full_version < '3.11' + - inline-snapshot>=0.24 ; extra == 'dev' + - jsonschema>=4 ; extra == 'dev' + - mypy>=1.14.1 ; extra == 'dev' + - nodeenv>=1.9.1 ; extra == 'dev' + - packaging>=25 ; extra == 'dev' + - pre-commit>=3.5 ; extra == 'dev' + - pytest>=8.3.5 ; extra == 'dev' + - pytest-cov>=5 ; extra == 'dev' + - rich-codex>=1.2.11 ; extra == 'dev' + - ruff>=0.12.4 ; extra == 'dev' + - typer>=0.15,<0.26 ; extra == 'dev' + - types-setuptools>=75.8.0.20250110 ; extra == 'dev' + - markdown-include>=0.8.1 ; extra == 'docs' + - mike>=2.1.3 ; extra == 'docs' + - mkdocs[docs]>=1.6.1 ; extra == 'docs' + - mkdocs-github-admonitions-plugin>=0.1.1 ; extra == 'docs' + - mkdocs-glightbox>=0.4 ; extra == 'docs' + - mkdocs-include-markdown-plugin>=7.1.7 ; python_full_version >= '3.9' and extra == 'docs' + - mkdocs-material[imaging]~=9.5.18 ; extra == 'docs' + - mkdocs-material-extensions>=1.3.1 ; extra == 'docs' + - mkdocs-redirects>=1.2.2 ; extra == 'docs' + - mkdocs-rss-plugin>=1.15 ; extra == 'docs' + - mkdocstrings[python]>=0.26.1 ; extra == 'docs' + - rich-codex>=1.2.11 ; extra == 'docs' + - typer>=0.15,<0.26 ; extra == 'docs' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/6e/55/51b92daaf6defb57f4dc56bdcce985400f75c6984a03ca5e78ccac717028/wrapt-2.3.0-cp312-cp312-macosx_11_0_arm64.whl + name: wrapt + version: 2.3.0 + sha256: 69e477046f2237ef0bc6547544ee73008dc764ca26eff44f09e976d221b34d5d + requires_dist: + - pytest ; extra == 'dev' + - setuptools ; extra == 'dev' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/6f/c5/7c16e99869e1f422629092cfd23e3b58e461988c3f9c36fd3624bb4142e6/parse-1.22.1-py2.py3-none-any.whl + name: parse + version: 1.22.1 + sha256: 20f0925a46f06602485ac90d751764d0697fd8455aaa97489ba8953a4b66de32 +- pypi: https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl + name: charset-normalizer + version: 3.4.9 + sha256: 45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl + name: aiohappyeyeballs + version: 2.7.1 + sha256: 9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/71/57/38c47753c67ad67f76ba04ea673c9b77431a19e7b2601937e6872a99e841/jsonasobj-1.3.1-py3-none-any.whl + name: jsonasobj + version: 1.3.1 + sha256: b9e329dc1ceaae7cf5d5b214684a0b100e0dad0be6d5bbabac281ec35ddeca65 +- pypi: https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl + name: sphinx + version: 9.1.0 + sha256: c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978 + requires_dist: + - sphinxcontrib-applehelp>=1.0.7 + - sphinxcontrib-devhelp>=1.0.6 + - sphinxcontrib-htmlhelp>=2.0.6 + - sphinxcontrib-jsmath>=1.0.1 + - sphinxcontrib-qthelp>=1.0.6 + - sphinxcontrib-serializinghtml>=1.1.9 + - jinja2>=3.1 + - pygments>=2.17 + - docutils>=0.21,<0.23 + - snowballstemmer>=2.2 + - babel>=2.13 + - alabaster>=0.7.14 + - imagesize>=1.3 + - requests>=2.30.0 + - roman-numerals>=1.0.0 + - packaging>=23.0 + - colorama>=0.4.6 ; sys_platform == 'win32' + requires_python: '>=3.12' +- pypi: https://files.pythonhosted.org/packages/75/45/54bb2d8d4138964a94bef6e9afe48b0be4705ba66ac442ae7d8a8dc4ffef/click_option_group-0.5.9-py3-none-any.whl + name: click-option-group + version: 0.5.9 + sha256: ad2599248bd373e2e19bec5407967c3eec1d0d4fc4a5e77b08a0481e75991080 + requires_dist: + - click>=7.0 + - pre-commit ; extra == 'dev' + - pytest ; extra == 'dev' + - m2r2 ; extra == 'docs' + - pallets-sphinx-themes ; extra == 'docs' + - sphinx ; extra == 'docs' + - pytest ; extra == 'test' + - pytest ; extra == 'test-cov' + - pytest-cov ; extra == 'test-cov' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/77/2a/83d779d2dfb61f101d7b1c10073d18984e37262d8b4f171c99911a952430/virtualenv-21.7.3-py3-none-any.whl + name: virtualenv + version: 21.7.3 + sha256: 26dfda3c34f29bf1a3ca167426a67658d59979b9954e705aef60a5f724ce1773 + requires_dist: + - distlib>=0.3.7,<1 + - filelock>=3.24.2,<4 ; python_full_version >= '3.10' + - filelock>=3.16.1,<=3.19.1 ; python_full_version < '3.10' + - platformdirs>=3.9.1,<5 + - python-discovery>=1.4.2 + - typing-extensions>=4.13.2 ; python_full_version < '3.11' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl + name: babel + version: 2.18.0 + sha256: e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35 + requires_dist: + - pytz>=2015.7 ; python_full_version < '3.9' + - tzdata ; sys_platform == 'win32' and extra == 'dev' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'dev' + - freezegun~=1.0 ; extra == 'dev' + - jinja2>=3.0 ; extra == 'dev' + - pytest-cov ; extra == 'dev' + - pytest>=6.0 ; extra == 'dev' + - pytz ; extra == 'dev' + - setuptools ; extra == 'dev' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/78/ac/5c5b959999b6f09c3026b5dfe171575bc3121c5236ce74f495096f25b203/greenlet-3.5.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl + name: greenlet + version: 3.5.5 + sha256: 147b25a42e5ca5be3d42356e8f608b37af715a1c196e9bf9d1627f3341adfe1d + requires_dist: + - sphinx ; extra == 'docs' + - furo ; extra == 'docs' + - objgraph ; extra == 'test' + - psutil ; extra == 'test' + - setuptools ; extra == 'test' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7b/24/61844afbf38acf419e01ca2639f7bd079584523d34471acbc4152ee991c5/hbreader-0.9.1-py3-none-any.whl + name: hbreader + version: 0.9.1 + sha256: 9a6e76c9d1afc1b977374a5dc430a1ebb0ea0488205546d4678d6e31cc5f6801 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl + name: rfc3339-validator + version: 0.1.4 + sha256: 24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa + requires_dist: + - six + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*' +- pypi: https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl + name: isoduration + version: 20.11.0 + sha256: b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042 + requires_dist: + - arrow>=0.15.0 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl + name: alabaster + version: 1.0.0 + sha256: fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl + name: httpcore + version: 1.0.9 + sha256: 2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 + requires_dist: + - certifi + - h11>=0.16 + - anyio>=4.0,<5.0 ; extra == 'asyncio' + - h2>=3,<5 ; extra == 'http2' + - socksio==1.* ; extra == 'socks' + - trio>=0.22.0,<1.0 ; extra == 'trio' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl + name: urllib3 + version: 2.7.0 + sha256: 9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + requires_dist: + - brotli>=1.2.0 ; platform_python_implementation == 'CPython' and extra == 'brotli' + - brotlicffi>=1.2.0.0 ; platform_python_implementation != 'CPython' and extra == 'brotli' + - h2>=4,<5 ; extra == 'h2' + - pysocks>=1.5.6,!=1.5.7,<2.0 ; extra == 'socks' + - backports-zstd>=1.0.0 ; python_full_version < '3.14' and extra == 'zstd' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl + name: rich + version: 15.0.0 + sha256: 33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb + requires_dist: + - ipywidgets>=7.5.1,<9 ; extra == 'jupyter' + - markdown-it-py>=2.2.0 + - pygments>=2.13.0,<3.0.0 + requires_python: '>=3.9.0' +- pypi: https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl + name: deprecated + version: 1.3.1 + sha256: 597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f + requires_dist: + - wrapt>=1.10,<3 + - inspect2 ; python_full_version < '3' + - tox ; extra == 'dev' + - pytest ; extra == 'dev' + - pytest-cov ; extra == 'dev' + - bump2version<1 ; extra == 'dev' + - setuptools ; python_full_version >= '3.12' and extra == 'dev' + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*' +- pypi: https://files.pythonhosted.org/packages/86/72/b03ca1560615933f079ba7d291d3532ed95c2a3205911fe71d192654acaa/shexjsg-0.9.0-py3-none-any.whl + name: shexjsg + version: 0.9.0 + sha256: abf18db2d9895bc46740f68ae699b2ccfe08c783f6e0c038e6077293ad01c0a5 + requires_dist: + - pyjsg>=0.12.3 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl + name: fastapi-cloud-cli + version: 0.23.0 + sha256: 1cd2ffa56e92e92c1fc63acc426c214dd928cbeed2a4c7c6a9a5fc85ea73de16 + requires_dist: + - typer>=0.16.0 + - uvicorn[standard]>=0.17.6 + - rignore>=0.5.1 + - httpx>=0.27.0 + - rich-toolkit>=0.20.3 + - pydantic[email]>=2.7.4 ; python_full_version < '3.13' + - pydantic[email]>=2.8.0 ; python_full_version == '3.13.*' + - pydantic[email]>=2.12.0 ; python_full_version >= '3.14' + - sentry-sdk>=2.20.0 + - fastar>=0.10.0 + - detect-installer>=0.1.0 + - uvicorn[standard]>=0.15.0 ; extra == 'standard' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl + name: nodeenv + version: 1.10.0 + sha256: 5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*' +- pypi: https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl + name: prefixmaps + version: 0.2.6 + sha256: f6cef28a7320fc6337cf411be212948ce570333a0ce958940ef684c7fb192a62 + requires_dist: + - curies>=0.5.3 + - pyyaml>=5.3.1 + requires_python: '>=3.8,<4.0' +- pypi: https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl + name: graphviz + version: '0.21' + sha256: 54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42 + requires_dist: + - build ; extra == 'dev' + - wheel ; extra == 'dev' + - twine ; extra == 'dev' + - flake8 ; extra == 'dev' + - flake8-pyproject ; extra == 'dev' + - pep8-naming ; extra == 'dev' + - tox>=3 ; extra == 'dev' + - pytest>=7,<8.1 ; extra == 'test' + - pytest-mock>=3 ; extra == 'test' + - pytest-cov ; extra == 'test' + - coverage ; extra == 'test' + - sphinx>=5,<7 ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - sphinx-rtd-theme>=0.2.5 ; extra == 'docs' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl + name: identify + version: 2.6.19 + sha256: 20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a + requires_dist: + - ukkonen ; extra == 'license' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl + name: annotated-types + version: 0.8.0 + sha256: f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl + name: markupsafe + version: 3.0.3 + sha256: 1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl + name: jsonpointer + version: 3.1.1 + sha256: 8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl + name: requests + version: 2.34.2 + sha256: 2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 + requires_dist: + - charset-normalizer>=2,<4 + - idna>=2.5,<4 + - urllib3>=1.26,<3 + - certifi>=2023.5.7 + - pysocks>=1.5.6,!=1.5.7 ; extra == 'socks' + - chardet>=3.0.2,<8 ; extra == 'use-chardet-on-py3' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl + name: rpds-py + version: 2026.6.3 + sha256: 538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/a4/d1/9725ec62421dcfe430d874cba9f82c0288baee0c4b7dd8aa8dcca2b0394e/fastapi_pagination-0.15.16-py3-none-any.whl + name: fastapi-pagination + version: 0.15.16 + sha256: 86dc73620812d47c297a7b8baca4eba2bac5d3f1d73aa75dc6e3bb12c0b803f7 + requires_dist: + - fastapi>=0.93.0 + - pydantic>=1.9.1 + - typing-extensions>=4.8.0 + - aiosqlite>=0.19.0,!=0.22.0,!=0.22.1 ; extra == 'aiosqlite' + - asyncpg>=0.24.0 ; extra == 'asyncpg' + - sqlalchemy>=1.3.20 ; extra == 'asyncpg' + - beanie>=2.1.0 ; extra == 'beanie' + - bunnet>=1.1.0 ; extra == 'bunnet' + - databases>=0.6.0 ; extra == 'databases' + - databases>=0.6.0 ; extra == 'django' + - django<6.0.0 ; extra == 'django' + - elasticsearch-dsl>=8.13.0 ; extra == 'elasticsearch' + - google-cloud-firestore>=2.19.0 ; extra == 'firestore' + - mongoengine>=0.23.1 ; extra == 'mongoengine' + - motor>=3.6.0 ; extra == 'motor' + - odmantic>=1.0.2 ; extra == 'odmantic' + - databases>=0.6.0 ; extra == 'orm' + - orm>=0.3.1 ; extra == 'orm' + - ormar>=0.21.0 ; extra == 'ormar' + - aiosqlite>=0.19.0,!=0.22.0,!=0.22.1 ; extra == 'peewee' + - greenlet ; extra == 'peewee' + - peewee>=4.0 ; extra == 'peewee' + - peewee>=4.0 ; extra == 'peewee-sync' + - piccolo>=0.89 ; extra == 'piccolo' + - psycopg[binary]>=3.3.2 ; extra == 'psycopg' + - scylla-driver>=3.25.6 ; extra == 'scylla-driver' + - sqlakeyset>=2.0.1680321678 ; extra == 'sqlalchemy' + - sqlalchemy>=1.3.20 ; extra == 'sqlalchemy' + - sqlakeyset>=2.0.1680321678 ; extra == 'sqlmodel' + - sqlmodel>=0.0.22 ; extra == 'sqlmodel' + - tortoise-orm>=0.22.0 ; extra == 'tortoise' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a5/42/cc4feea2945cb3051038f090c9b36bd5b8a9d7f5a894a506a8983e33fd1c/httptools-0.8.0-cp312-cp312-macosx_11_0_arm64.whl + name: httptools + version: 0.8.0 + sha256: 5931891fb7b441b8a3853cf1b85c82c903defce084dd5f6771ca46e31bf862c5 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl + name: pyyaml + version: 6.0.2 + sha256: ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl + name: multidict + version: 6.7.1 + sha256: b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7 + requires_dist: + - typing-extensions>=4.1.0 ; python_full_version < '3.11' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a9/a8/c6e6db62226df8cd3d950889925d826a0e35ac567cc5df548a872944f1ed/rignore-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: rignore + version: 0.8.1 + sha256: 2e68cfc4ee0a2909952af2aebd608d1cd22f7d1cdce332cb0b5ea3762939865d + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl + name: mdurl + version: 0.1.2 + sha256: 84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl + name: markdown-it-py + version: 4.2.0 + sha256: 9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + requires_dist: + - mdurl~=0.1 + - psutil ; extra == 'benchmarking' + - pytest ; extra == 'benchmarking' + - pytest-benchmark ; extra == 'benchmarking' + - commonmark~=0.9 ; extra == 'compare' + - markdown~=3.4 ; extra == 'compare' + - mistletoe~=1.0 ; extra == 'compare' + - mistune~=3.0 ; extra == 'compare' + - panflute~=2.3 ; extra == 'compare' + - markdown-it-pyrs ; extra == 'compare' + - linkify-it-py>=1,<3 ; extra == 'linkify' + - mdit-py-plugins>=0.5.0 ; extra == 'plugins' + - gprof2dot ; extra == 'profiling' + - mdit-py-plugins>=0.5.0 ; extra == 'rtd' + - myst-parser ; extra == 'rtd' + - pyyaml ; extra == 'rtd' + - sphinx ; extra == 'rtd' + - sphinx-copybutton ; extra == 'rtd' + - sphinx-design ; extra == 'rtd' + - sphinx-book-theme~=1.0 ; extra == 'rtd' + - jupyter-sphinx ; extra == 'rtd' + - ipykernel ; extra == 'rtd' + - coverage ; extra == 'testing' + - pytest ; extra == 'testing' + - pytest-cov ; extra == 'testing' + - pytest-regressions ; extra == 'testing' + - pytest-timeout ; extra == 'testing' + - requests ; extra == 'testing' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl + name: watchdog + version: 6.0.0 + sha256: 20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2 + requires_dist: + - pyyaml>=3.10 ; extra == 'watchmedo' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + name: six + version: 1.17.0 + sha256: 4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pyyaml + version: 6.0.2 + sha256: 80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl + name: dnspython + version: 2.8.0 + sha256: 01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af + requires_dist: + - black>=25.1.0 ; extra == 'dev' + - coverage>=7.0 ; extra == 'dev' + - flake8>=7 ; extra == 'dev' + - hypercorn>=0.17.0 ; extra == 'dev' + - mypy>=1.17 ; extra == 'dev' + - pylint>=3 ; extra == 'dev' + - pytest-cov>=6.2.0 ; extra == 'dev' + - pytest>=8.4 ; extra == 'dev' + - quart-trio>=0.12.0 ; extra == 'dev' + - sphinx-rtd-theme>=3.0.0 ; extra == 'dev' + - sphinx>=8.2.0 ; extra == 'dev' + - twine>=6.1.0 ; extra == 'dev' + - wheel>=0.45.0 ; extra == 'dev' + - cryptography>=45 ; extra == 'dnssec' + - h2>=4.2.0 ; extra == 'doh' + - httpcore>=1.0.0 ; extra == 'doh' + - httpx>=0.28.0 ; extra == 'doh' + - aioquic>=1.2.0 ; extra == 'doq' + - idna>=3.10 ; extra == 'idna' + - trio>=0.30 ; extra == 'trio' + - wmi>=1.5.1 ; sys_platform == 'win32' and extra == 'wmi' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl + name: openpyxl + version: 3.1.5 + sha256: 5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2 + requires_dist: + - et-xmlfile + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl + name: et-xmlfile + version: 2.0.0 + sha256: 7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl + name: filelock + version: 3.32.2 + sha256: 87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl + name: sphinxcontrib-jsmath + version: 1.0.1 + sha256: 2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 + requires_dist: + - pytest ; extra == 'test' + - flake8 ; extra == 'test' + - mypy ; extra == 'test' + requires_python: '>=3.5' +- pypi: https://files.pythonhosted.org/packages/c4/8e/847935c588455b0d82fa57a5a8ced4c73a928e30f2012639228e566e3283/chardet-7.5.1-cp312-cp312-macosx_11_0_arm64.whl + name: chardet + version: 7.5.1 + sha256: 8a001a8f030625b705d9a4e68116e573462bd38192cc6c1bfa318b45606747ac + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c7/d5/68e6e9bca63c0badf67002890a46d3784c958de45b65e1275ec583ca1f06/uvicorn-0.52.1-py3-none-any.whl + name: uvicorn + version: 0.52.1 + sha256: e4403f9d93188cf9d1088e9f40e3acd12630e2df8675316704379a7fc20fff6a + requires_dist: + - click>=7.0 + - h11>=0.8 + - typing-extensions>=4.0 ; python_full_version < '3.11' + - httptools>=0.8.0 ; extra == 'standard' + - python-dotenv>=0.13 ; extra == 'standard' + - pyyaml>=5.1 ; extra == 'standard' + - uvloop>=0.15.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32' and extra == 'standard' + - watchfiles>=0.20 ; extra == 'standard' + - websockets>=13.0 ; extra == 'standard' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl + name: starlette + version: 1.6.0 + sha256: a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c + requires_dist: + - anyio>=3.6.2,<5 + - typing-extensions>=4.10.0 ; python_full_version < '3.13' + - httpx2>=2.0.0 ; extra == 'full' + - httpx>=0.27.0,<0.29.0 ; extra == 'full' + - itsdangerous ; extra == 'full' + - jinja2 ; extra == 'full' + - python-multipart>=0.0.18 ; extra == 'full' + - pyyaml ; extra == 'full' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/c9/13/c51c203d05c765e29b83006eba89bfec3f41e72407668f8beee482153e9f/datasalad-0.9.0-py3-none-any.whl + name: datasalad + version: 0.9.0 + sha256: 63b58f4039692316ac02c57a6f0a9da0a9157f9578e1e9efc91b38ddd614dd50 + requires_dist: + - sphinx ; extra == 'docs' + - sphinx-autodoc-typehints ; extra == 'docs' + - sphinx-rtd-theme ; extra == 'docs' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl + name: fastapi + version: 0.141.1 + sha256: bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3 + requires_dist: + - starlette>=0.46.0 + - pydantic>=2.9.0 + - typing-extensions>=4.8.0 + - typing-inspection>=0.4.2 + - annotated-doc>=0.0.2 + - fastapi-cli[standard]>=0.0.32 ; extra == 'standard' + - fastar>=0.9.0 ; extra == 'standard' + - httpx>=0.23.0,<1.0.0 ; extra == 'standard' + - jinja2>=3.1.5 ; extra == 'standard' + - python-multipart>=0.0.18 ; extra == 'standard' + - email-validator>=2.0.0 ; extra == 'standard' + - uvicorn[standard]>=0.12.0 ; extra == 'standard' + - pydantic-settings>=2.0.0 ; extra == 'standard' + - pydantic-extra-types>=2.0.0 ; extra == 'standard' + - fastapi-cli[standard-no-fastapi-cloud-cli]>=0.0.32 ; extra == 'standard-no-fastapi-cloud-cli' + - httpx>=0.23.0,<1.0.0 ; extra == 'standard-no-fastapi-cloud-cli' + - jinja2>=3.1.5 ; extra == 'standard-no-fastapi-cloud-cli' + - python-multipart>=0.0.18 ; extra == 'standard-no-fastapi-cloud-cli' + - email-validator>=2.0.0 ; extra == 'standard-no-fastapi-cloud-cli' + - uvicorn[standard]>=0.12.0 ; extra == 'standard-no-fastapi-cloud-cli' + - pydantic-settings>=2.0.0 ; extra == 'standard-no-fastapi-cloud-cli' + - pydantic-extra-types>=2.0.0 ; extra == 'standard-no-fastapi-cloud-cli' + - fastapi-cli[standard]>=0.0.32 ; extra == 'all' + - httpx>=0.23.0,<1.0.0 ; extra == 'all' + - jinja2>=3.1.5 ; extra == 'all' + - python-multipart>=0.0.18 ; extra == 'all' + - itsdangerous>=1.1.0 ; extra == 'all' + - pyyaml>=5.3.1 ; extra == 'all' + - email-validator>=2.0.0 ; extra == 'all' + - uvicorn[standard]>=0.12.0 ; extra == 'all' + - pydantic-settings>=2.0.0 ; extra == 'all' + - pydantic-extra-types>=2.0.0 ; extra == 'all' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/cb/51/3e7e021920cfe2f7d18b672642e13f7dc4f53545d530b52ee6533b6681ca/CFGraph-0.2.1.tar.gz + name: cfgraph + version: 0.2.1 + sha256: b57fe7044a10b8ff65aa3a8a8ddc7d4cd77bf511b42e57289cd52cbc29f8fe74 + requires_dist: + - rdflib>=0.4.2 +- pypi: https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl + name: iniconfig + version: 2.3.0 + sha256: f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl + name: detect-installer + version: 0.1.0 + sha256: 034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/cc/a6/d5e2a4e48495616440a21eed07558219ca90243ad00b0502586f95bd4833/fastar-0.11.0-cp312-cp312-macosx_11_0_arm64.whl + name: fastar + version: 0.11.0 + sha256: 0d9d6b052baf5380baea866675dab6ccd04ec2460d12b1c46f10ce3f4ee6a820 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl + name: fqdn + version: 1.5.1 + sha256: 3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014 + requires_dist: + - cached-property>=1.3.0 ; python_full_version < '3.8' + requires_python: '>=2.7,!=3.0,!=3.1,!=3.2,!=3.3,!=3.4,<4' +- pypi: https://files.pythonhosted.org/packages/d0/d2/760527679057a7dad67f4e41f3e0c463b247f0bdbffc594e0add7c9077d6/rdflib_jsonld-0.6.1-py2.py3-none-any.whl + name: rdflib-jsonld + version: 0.6.1 + sha256: bcf84317e947a661bae0a3f2aee1eced697075fc4ac4db6065a3340ea0f10fc2 + requires_dist: + - rdflib>=5.0.0 +- pypi: https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl + name: fastapi-cli + version: 0.0.32 + sha256: 8dcc286fa32f01bbd3f65dd09cfd5a2540ed5f2230b77db7fd30978d6165f3c4 + requires_dist: + - typer>=0.16.0 + - uvicorn[standard]>=0.15.0 + - rich-toolkit>=0.14.8 + - tomli>=2.0.0 ; python_full_version < '3.11' + - uvicorn[standard]>=0.15.0 ; extra == 'standard' + - fastapi-cloud-cli>=0.1.1 ; extra == 'standard' + - uvicorn[standard]>=0.15.0 ; extra == 'standard-no-fastapi-cloud-cli' + - fastapi-new>=0.0.2 ; extra == 'new' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: yarl + version: 1.24.5 + sha256: f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9 + requires_dist: + - idna>=2.0 + - multidict>=4.0 + - propcache>=0.2.1 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/d9/82/123660edc759c225626b3b91952c7625f85c77a8362acbc35a4623120f7d/websockets-17.0.1-cp312-cp312-macosx_11_0_arm64.whl + name: websockets + version: 17.0.1 + sha256: c23e532c8a2325a1e7486de8763a60dc43e83f01bcaeca07e3ba79652c156db1 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl + name: anyio + version: 4.14.2 + sha256: 9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 + requires_dist: + - exceptiongroup>=1.0.2 ; python_full_version < '3.11' + - idna>=2.8 + - typing-extensions>=4.5 ; python_full_version < '3.13' + - trio>=0.32.0 ; extra == 'trio' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl + name: cfgv + version: 3.5.0 + sha256: a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/dc/1e/fb11174c9eaebcec27d36e9e994b90ffa168bc3226925900b9dbbf16c9da/pytest-logging-2015.11.4.tar.gz + name: pytest-logging + version: 2015.11.4 + sha256: cec5c85ecf18aab7b2ead5498a31b9f758680ef5a902b9054ab3f2bdbb77c896 + requires_dist: + - pytest>=2.8.1 +- pypi: https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl + name: email-validator + version: 2.3.0 + sha256: 80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4 + requires_dist: + - dnspython>=2.0.0 + - idna>=2.0.0 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl + name: shellingham + version: 1.5.4 + sha256: 7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl + name: python-multipart + version: 0.0.32 + sha256: ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl + name: webcolors + version: 25.10.0 + sha256: 032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl + name: rich-toolkit + version: 0.20.3 + sha256: 419aa87516d5f3849cca553c6dcf707c02a36d508fcf996946606725d34a3002 + requires_dist: + - click>=8.1.7 + - rich>=13.7.1 + - typing-extensions>=4.12.2 + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl + name: httptools + version: 0.8.0 + sha256: b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl + name: tzdata + version: '2026.3' + sha256: dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931 + requires_python: '>=2' +- pypi: https://files.pythonhosted.org/packages/e5/90/0d93963711f811efe528e3cead2f2bfb78c196df74d8a24fe8d655288e50/jsonasobj2-1.0.4-py3-none-any.whl + name: jsonasobj2 + version: 1.0.4 + sha256: 12e86f86324d54fcf60632db94ea74488d5314e3da554c994fe1e2c6f29acb79 + requires_dist: + - hbreader + requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl + name: uri-template + version: 1.3.0 + sha256: a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363 + requires_dist: + - types-pyyaml ; extra == 'dev' + - mypy ; extra == 'dev' + - flake8 ; extra == 'dev' + - flake8-annotations ; extra == 'dev' + - flake8-bandit ; extra == 'dev' + - flake8-bugbear ; extra == 'dev' + - flake8-commas ; extra == 'dev' + - flake8-comprehensions ; extra == 'dev' + - flake8-continuation ; extra == 'dev' + - flake8-datetimez ; extra == 'dev' + - flake8-docstrings ; extra == 'dev' + - flake8-import-order ; extra == 'dev' + - flake8-literal ; extra == 'dev' + - flake8-modern-annotations ; extra == 'dev' + - flake8-noqa ; extra == 'dev' + - flake8-pyproject ; extra == 'dev' + - flake8-requirements ; extra == 'dev' + - flake8-typechecking-import ; extra == 'dev' + - flake8-use-fstring ; extra == 'dev' + - pep8-naming ; extra == 'dev' + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/e8/3d/1087453384dbde46a8c7f9356eead2c58be8a7bf156bca40243377c85715/more_itertools-11.1.0-py3-none-any.whl + name: more-itertools + version: 11.1.0 + sha256: 4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl + name: yarl + version: 1.24.5 + sha256: 9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c + requires_dist: + - idna>=2.0 + - multidict>=4.0 + - propcache>=0.2.1 + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl + name: python-dateutil + version: 2.9.0.post0 + sha256: a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + requires_dist: + - six>=1.5 + requires_python: '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*' +- pypi: https://files.pythonhosted.org/packages/ed/7e/e7ceaf949ecee97ba6003872b3d88bed1b22941742381030d9fc8ea4525e/pystow-0.9.0-py3-none-any.whl + name: pystow + version: 0.9.0 + sha256: 5d33926cea3c03c60731df6ffbe9b9467b11d5b1d280f13201874b80fc5f4ba2 + requires_dist: + - tqdm + - typing-extensions + - backports-zstd ; python_full_version < '3.14' + - boto3 ; extra == 'aws' + - bs4 ; extra == 'bs4' + - requests ; extra == 'bs4' + - click ; extra == 'cli' + - pandas ; extra == 'pandas' + - pydantic ; extra == 'pydantic' + - ratelimit ; extra == 'ratelimit' + - requests ; extra == 'ratelimit' + - rdflib ; extra == 'rdf' + - requests ; extra == 'requests' + - lxml ; extra == 'xml' + - pyyaml ; extra == 'yaml' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl + name: arrow + version: 1.4.0 + sha256: 749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 + requires_dist: + - python-dateutil>=2.7.0 + - backports-zoneinfo==0.2.1 ; python_full_version < '3.9' + - tzdata ; python_full_version >= '3.9' + - doc8 ; extra == 'doc' + - sphinx>=7.0.0 ; extra == 'doc' + - sphinx-autobuild ; extra == 'doc' + - sphinx-autodoc-typehints ; extra == 'doc' + - sphinx-rtd-theme>=1.3.0 ; extra == 'doc' + - dateparser==1.* ; extra == 'test' + - pre-commit ; extra == 'test' + - pytest ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytz==2025.2 ; extra == 'test' + - simplejson==3.* ; extra == 'test' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl + name: multidict + version: 6.7.1 + sha256: bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961 + requires_dist: + - typing-extensions>=4.1.0 ; python_full_version < '3.11' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl + name: pygments + version: 2.20.0 + sha256: 81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + requires_dist: + - colorama>=0.4.6 ; extra == 'windows-terminal' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl + name: tqdm + version: 4.70.0 + sha256: 7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953 + requires_dist: + - colorama ; sys_platform == 'win32' + - requests ; extra == 'discord' + - envwrap ; extra == 'discord' + - slack-sdk ; extra == 'slack' + - envwrap ; extra == 'slack' + - requests ; extra == 'telegram' + - envwrap ; extra == 'telegram' + - ipywidgets>=6 ; extra == 'notebook' + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl + name: aiosignal + version: 1.4.0 + sha256: 053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e + requires_dist: + - frozenlist>=1.1.0 + - typing-extensions>=4.2 ; python_full_version < '3.13' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl + name: click + version: 8.4.2 + sha256: e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + requires_dist: + - colorama ; sys_platform == 'win32' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl + name: fsspec + version: 2026.7.0 + sha256: b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 + requires_dist: + - adlfs ; extra == 'abfs' + - adlfs ; extra == 'adl' + - pyarrow>=1 ; extra == 'arrow' + - dask ; extra == 'dask' + - distributed ; extra == 'dask' + - pre-commit ; extra == 'dev' + - ruff>=0.5 ; extra == 'dev' + - numpydoc ; extra == 'doc' + - sphinx ; extra == 'doc' + - sphinx-design ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - yarl ; extra == 'doc' + - dropbox ; extra == 'dropbox' + - dropboxdrivefs ; extra == 'dropbox' + - requests ; extra == 'dropbox' + - adlfs ; extra == 'full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'full' + - dask ; extra == 'full' + - distributed ; extra == 'full' + - dropbox ; extra == 'full' + - dropboxdrivefs ; extra == 'full' + - fusepy ; extra == 'full' + - gcsfs>=2026.4.0 ; extra == 'full' + - libarchive-c ; extra == 'full' + - ocifs ; extra == 'full' + - panel ; extra == 'full' + - paramiko ; extra == 'full' + - pyarrow>=1 ; extra == 'full' + - pygit2 ; extra == 'full' + - requests ; extra == 'full' + - s3fs>=2026.6.0 ; extra == 'full' + - smbprotocol ; extra == 'full' + - tqdm ; extra == 'full' + - fusepy ; extra == 'fuse' + - gcsfs>=2026.4.0 ; extra == 'gcs' + - pygit2 ; extra == 'git' + - requests ; extra == 'github' + - gcsfs>=2026.4.0 ; extra == 'gs' + - panel ; extra == 'gui' + - pyarrow>=1 ; extra == 'hdfs' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'http' + - libarchive-c ; extra == 'libarchive' + - ocifs ; extra == 'oci' + - s3fs>=2026.6.0 ; extra == 's3' + - paramiko ; extra == 'sftp' + - smbprotocol ; extra == 'smb' + - paramiko ; extra == 'ssh' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test' + - numpy ; extra == 'test' + - pytest ; extra == 'test' + - pytest-asyncio!=0.22.0 ; extra == 'test' + - pytest-benchmark ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytest-recording ; extra == 'test' + - pytest-rerunfailures ; extra == 'test' + - requests ; extra == 'test' + - aiobotocore>=2.5.4,<3.0.0 ; extra == 'test-downstream' + - dask[dataframe,test] ; extra == 'test-downstream' + - moto[server]>4,<5 ; extra == 'test-downstream' + - pytest-timeout ; extra == 'test-downstream' + - xarray ; extra == 'test-downstream' + - adlfs ; extra == 'test-full' + - aiohttp!=4.0.0a0,!=4.0.0a1 ; extra == 'test-full' + - backports-zstd ; python_full_version < '3.14' and extra == 'test-full' + - cloudpickle ; extra == 'test-full' + - dask ; extra == 'test-full' + - distributed ; extra == 'test-full' + - dropbox ; extra == 'test-full' + - dropboxdrivefs ; extra == 'test-full' + - fastparquet ; extra == 'test-full' + - fusepy ; extra == 'test-full' + - gcsfs>=2026.4.0 ; extra == 'test-full' + - jinja2 ; extra == 'test-full' + - kerchunk ; extra == 'test-full' + - libarchive-c ; extra == 'test-full' + - lz4 ; extra == 'test-full' + - notebook ; extra == 'test-full' + - numpy ; extra == 'test-full' + - ocifs ; extra == 'test-full' + - pandas<3.0.0 ; extra == 'test-full' + - panel ; extra == 'test-full' + - paramiko ; extra == 'test-full' + - pyarrow>=1 ; extra == 'test-full' + - pyftpdlib ; extra == 'test-full' + - pygit2 ; extra == 'test-full' + - pytest ; extra == 'test-full' + - pytest-asyncio!=0.22.0 ; extra == 'test-full' + - pytest-benchmark ; extra == 'test-full' + - pytest-cov ; extra == 'test-full' + - pytest-mock ; extra == 'test-full' + - pytest-recording ; extra == 'test-full' + - pytest-rerunfailures ; extra == 'test-full' + - python-snappy ; extra == 'test-full' + - requests ; extra == 'test-full' + - s3fs>=2026.6.0 ; extra == 'test-full' + - smbprotocol ; extra == 'test-full' + - tqdm ; extra == 'test-full' + - urllib3 ; extra == 'test-full' + - zarr<3.2.0 ; extra == 'test-full' + - zstandard ; python_full_version < '3.14' and extra == 'test-full' + - tqdm ; extra == 'tqdm' + requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl + name: pydantic + version: 2.13.4 + sha256: 45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba + requires_dist: + - annotated-types>=0.6.0 + - pydantic-core==2.46.4 + - typing-extensions>=4.14.1 + - typing-inspection>=0.4.2 + - email-validator>=2.0.0 ; extra == 'email' + - tzdata ; python_full_version >= '3.9' and sys_platform == 'win32' and extra == 'timezone' + requires_python: '>=3.9' diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 0000000..ad912c3 --- /dev/null +++ b/pixi.toml @@ -0,0 +1,75 @@ +[workspace] +name = "orinoco-full-con-migration" +channels = ["conda-forge"] +platforms = [{ platform = "osx-arm64", macos = "14.0" }, "linux-64"] + +[dependencies] +python = ">=3.12,<3.13" +hugo = "==0.154.5" +nodejs = ">=22,<23" + +[target.linux-64.dependencies] +git-annex = "==10.20260601" + +[target.osx-arm64-macos-14-0.pypi-dependencies] +git-annex = "==10.20260601" + +[tasks] +checkout-submodules = "python3 tools/checkout_submodules.py" +build = { depends-on = ["checkout-submodules"], cmd = "python3 tools/build_con_site.py" } +verify-static = { depends-on = ["checkout-submodules"], cmd = "python3 tools/build_con_site.py --repeat-destination build/con-site-repeat" } +build-con-project-path = { depends-on = ["checkout-submodules"], cmd = "python3 tools/build_con_site.py --destination build/con-site-project --base-url http://127.0.0.1:8767/full-con-migration/" } +serve = { depends-on = ["build", "prepare-local-stack"], cmd = "tools/serve_local_stack.sh" } +serve-static = { depends-on = ["build"], cmd = "python3 -m http.server 8767 --directory build/con-site" } +render-con-projection = { depends-on = ["checkout-submodules"], cmd = "python3 tools/con_projection.py render" } +update-con-projection = { depends-on = ["checkout-submodules"], cmd = "python3 tools/con_projection.py update" } +verify-con-projection = { depends-on = ["checkout-submodules"], cmd = "python3 tools/con_projection.py verify" } +check-con-projection = { depends-on = ["checkout-submodules"], cmd = "python3 tools/con_projection.py check-snapshot" } +update-con-assembly = { depends-on = ["checkout-submodules"], cmd = "python3 tools/build_con_site.py --update-assembly-manifest" } +verify-con-assembly = { depends-on = ["checkout-submodules"], cmd = "python3 tools/build_con_site.py --check-assembly-manifest" } +hydrate-con-assets = { depends-on = ["checkout-submodules"], cmd = "python3 tools/con_assets.py" } +build-upstream = { depends-on = ["checkout-submodules"], cmd = "tools/build_upstream_site.sh", env = { BASE_URL = "http://127.0.0.1:8768/", DESTINATION = "build/upstream-local", SHACL_VUE_URL = "http://127.0.0.1:3000/" } } +serve-upstream = { depends-on = ["build-upstream"], cmd = "python3 -m http.server 8768 --directory build/upstream-local" } +build-pages-editor = { depends-on = ["build-pool-ui"], cmd = "python3 tools/build_pages_editor.py" } +build-pages = { depends-on = ["checkout-submodules", "build-pages-editor"], cmd = "python3 tools/build_con_pages.py --require-editor" } +verify-pages-editor = { depends-on = ["install-pool-ui"], cmd = "python3 tools/build_pages_editor.py --repeat-destination build/pages-editor-repeat" } +verify-pages = { depends-on = ["checkout-submodules", "verify-pages-editor"], cmd = "python3 tools/build_con_pages.py --require-editor --repeat-destination build/pages-preview-repeat/orinoco-lite-dev" } +serve-pages = { depends-on = ["build-pages"], cmd = "python3 -m http.server 8766 --directory build/pages-preview" } +audit-pages = { cmd = "python3 tools/build_con_pages.py --require-editor --check-only" } +test-pages = "python3 -m unittest tests.test_con_pages tests.test_pages_editor" +review-editor-bundle = "python3 tools/apply_editor_bundle.py" +prepare-local-stack = { depends-on = ["checkout-submodules", "build-pool-ui"], cmd = "python3 tools/prepare_local_stack.py" } +serve-dump-things = { depends-on = ["prepare-local-stack"], cmd = "tools/serve_local_dumpthings.sh" } +serve-git-annex = { depends-on = ["prepare-local-stack"], cmd = "python3 tools/serve_local_gitannex.py" } +seed-local-pool = { depends-on = ["prepare-local-stack"], cmd = "python3 tools/seed_local_pool.py" } +refresh-local-pool = { depends-on = ["checkout-submodules", "build-pool-ui"], cmd = "REFRESH_UPSTREAM_POOL=1 python3 tools/prepare_local_stack.py" } +install-pool-ui = { depends-on = ["checkout-submodules"], cmd = "npm --prefix submodules/pool.psychoinformatics.de-ui/shacl-vue ci" } +build-pool-ui = { depends-on = ["install-pool-ui"], cmd = "make -C submodules/pool.psychoinformatics.de-ui build-ui" } +serve-shacl-vue = { depends-on = ["prepare-local-stack"], cmd = "python3 -m http.server 3000 --directory build/local-stack/ui" } +check-local-stack = { cmd = "python3 tools/check_local_stack.py" } +format-docs = { cmd = "snapper --in-place README.md AGENTS.md docs/*.md" } +install-hooks = { cmd = "git config core.hooksPath .githooks" } +check-format = { cmd = "pre-commit run --all-files" } +test = "python3 -m unittest discover -s tests -p 'test_*.py'" +install-browser-tests = { cmd = "npm ci && npx playwright install chromium webkit", env = { PLAYWRIGHT_BROWSERS_PATH = "build/playwright-browsers" } } +install-pages-browser-tests = { cmd = "npm ci && npx playwright install chromium", env = { PLAYWRIGHT_BROWSERS_PATH = "build/playwright-browsers" } } +test-pages-browser = { depends-on = ["verify-pages", "install-pages-browser-tests"], cmd = "npx playwright test tests/browser/static-editor.spec.mjs --project=chromium", env = { PLAYWRIGHT_BROWSERS_PATH = "build/playwright-browsers", PLAYWRIGHT_STATIC_ONLY = "1" } } +test-browser = { depends-on = ["build", "build-upstream", "build-con-project-path", "build-pages", "prepare-local-stack", "install-browser-tests"], cmd = "npm run test:browser", env = { PLAYWRIGHT_BROWSERS_PATH = "build/playwright-browsers" } } +test-all = { depends-on = ["test", "verify-con-projection", "verify-con-assembly", "verify-static", "test-browser"] } + +[pypi-dependencies] +snapper-fmt = "==0.8.1" +pre-commit = "==4.3.0" +dump-things-service = { path = "submodules/dump-things-service" } +dump-things-pyclient = { path = "submodules/dump-things-pyclient" } +query-things = { path = "submodules/query-things" } +jinja2 = "==3.1.6" +packaging = "==26.3" +pyyaml = "==6.0.2" +linkml = "==1.11.1" +linkml-runtime = "==1.11.1" +pydantic = "==2.13.4" +rdflib = "==7.6.0" + +[pypi-options.dependency-overrides] +dump-things-pyclient = { path = "submodules/dump-things-pyclient" } diff --git a/playwright.config.mjs b/playwright.config.mjs new file mode 100644 index 0000000..8a9180a --- /dev/null +++ b/playwright.config.mjs @@ -0,0 +1,49 @@ +import { defineConfig, devices } from '@playwright/test'; + +const buildRoot = 'build/playwright'; +const staticOnly = process.env.PLAYWRIGHT_STATIC_ONLY === '1'; + +export default defineConfig({ + testDir: './tests/browser', + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 90_000, + expect: { + timeout: 15_000, + }, + reporter: [ + ['list'], + ['html', { outputFolder: `${buildRoot}/report`, open: 'never' }], + ], + outputDir: `${buildRoot}/results`, + use: { + headless: true, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'off', + }, + webServer: staticOnly + ? undefined + : { + command: 'tools/serve_local_stack.sh', + url: 'http://127.0.0.1:8767/', + reuseExistingServer: false, + timeout: 240_000, + gracefulShutdown: { + signal: 'SIGTERM', + timeout: 15_000, + }, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + { + name: 'webkit', + testIgnore: '**/authenticated-editor.spec.mjs', + use: { ...devices['Desktop Safari'] }, + }, + ], +}); diff --git a/provenance/full-con-migration/baseline.yaml b/provenance/full-con-migration/baseline.yaml new file mode 100644 index 0000000..92f09ba --- /dev/null +++ b/provenance/full-con-migration/baseline.yaml @@ -0,0 +1,34 @@ +captured: 2026-08-11 +timezone: America/New_York + +accepted_checkpoint: + parent_branch: codex/clean-migration + parent_commit: f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d + site_branch: codex/clean-migration + site_commit: a122e506de9e4a13473edbe8d74a950d74032a16 + +successor: + parent_branch: codex/full-con-migration + site_branch: codex/full-con-migration + +upstream: + repository: https://hub.psychoinformatics.de/www/www-from-model.git + previous_base: 5b401e0c478a4409442b3a8a285bd3efd5d30e05 + reviewed_base: a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8 + review: >- + The range contains one CI-only deposit-changes workflow update and no + presentation-tree change. + +toolchain: + git_annex: "10.20260601" + osx_arm64_source: pypi-wheel + osx_minimum: "14.0" + linux_64_source: conda-forge + +migration_evidence: + legacy_site: + repository: https://github.com/con/centerforopenneuroscience.org.git + commit: e6e9200a0987a65097afff896105ff838be1659e + dump_research_info: + repository: https://github.com/con/dump-research-info.git + commit: 1c7e99ec6f296d5e6cb6a61e3b786227190802da diff --git a/provenance/upstream-psychoinformatics/baseline.yaml b/provenance/upstream-psychoinformatics/baseline.yaml new file mode 100644 index 0000000..1be417b --- /dev/null +++ b/provenance/upstream-psychoinformatics/baseline.yaml @@ -0,0 +1,175 @@ +captured: 2026-08-06 +timezone: America/New_York + +parent_trial: + repository: https://github.com/con/orinoco-lite-dev.git + branch: codex/upstream-psychoinformatics-trial + based_on_main_commit: 47bdf2f396e462d6622a166d2ba6c29f6a273b7c + isolated_worktree: /Users/johnlee/code/CON/orinoco-upstream-trial + +website: + upstream_repository: https://hub.psychoinformatics.de/www/www-from-model.git + upstream_commit: 5b401e0c478a4409442b3a8a285bd3efd5d30e05 + github_mirror: https://github.com/leej3/www-from-model.git + github_mirror_main: 6c8b9a5b7260dc20dfe1453dd863b353e8f90f06 + annex_metadata_commit: 010ca44f751d2ab60b9d4ad58c5931d1804e3c9e + congo_commit: 3623fa505ee42fee899844d94a4ff7f5a1ae9096 + previous_parent_pin: 6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce + changes_after_previous_pin: + - commit: 10087fa51ad34ca5c86334fc487ead660b190589 + date: 2026-08-04T01:05:55Z + subject: "chore: auto-generate content from metadata" + - commit: e50f88fd7dabdf5e00607d51360f3ad5ecfd328f + date: 2026-08-06T01:05:36Z + subject: "chore: auto-generate content from metadata" + - commit: 5b401e0c478a4409442b3a8a285bd3efd5d30e05 + date: 2026-08-06T11:21:48+02:00 + subject: "ci: use action from URL, not path" + +toolchain: + hugo: + version: 0.154.5 + edition: extended + tested_binary_sha256: 0e76db72687d544c79c4df8cb17d55a7e625eba943e76f5e9d73b1ac42b3a062 + local_git_annex: "10.20260420" + github_actions_git_annex: "10.20260717" + local_uv: 0.12.1 + github_actions: + checkout: 3d3c42e5aac5ba805825da76410c181273ba90b1 + setup_uv: c771a70e6277c0a99b617c7a806ffedaca235ff9 + setup_hugo: 2752ce1d29631191ea3f27c23495fa06139a5b78 + configure_pages: 45bfe0192ca1faeb007ade9deae92b16b8254a0d + upload_pages_artifact: fc324d3547104276b827a68afc52ff2a11cc49c9 + deploy_pages: cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 + +annex: + worktree_files: 39 + unique_keys: 38 + hydrated_bytes: 55916505 + web_available_files: 25 + upstream_annex_only_files: 14 + upstream_annex_only: + - path: assets/css/compiled/main.css + key: MD5E-s66312--fe858e6c7a5099d2af85e5bae653f1ef.css + - path: assets/img/logo.png + key: MD5E-s6193--c9902add4680efad545e73981469b50a.png + - path: assets/img/meerkat.png + key: MD5E-s60100--bacf03c8cb9ecfece98b09f3e06a7cad.png + - path: assets/img/meerkat_objective.png + key: MD5E-s97710--5c7aeb495fab557bff3e5ddd68823728.png + - path: assets/img/meerkat_person.png + key: MD5E-s96576--da283667dd896be1d22255c98e2bfce9.png + - path: assets/img/meerkat_project.png + key: MD5E-s84983--2d1a07b7c96f9d1b07c7440e0c6c025c.png + - path: assets/img/meerkat_topic.png + key: MD5E-s99320--d7f4c357a9f878fb3762db9900a99db0.png + - path: content/projects/trr379/logo.svg + key: VURL--https://hub.psychoinformatics.d-708490a0f80ddc1d3375fe4b46d1beb2 + - path: static/apple-touch-icon.png + key: MD5E-s2484--df15328c43aa65334af198edf4130ca5.png + - path: static/favicon-16x16.png + key: MD5E-s430--863eb1f06b7240006206c10721b9fafc.png + - path: static/favicon-32x32.png + key: MD5E-s812--32f43dbc38a18dcaf9fa80b19df4b8e9.png + - path: static/graph.js + key: MD5E-s182838--bf3056dcf2928be750c8483796e0c202.js + - path: static/graph.json + key: MD5E-s556448--7d509f65f046c12994a2062540cce03e.json + - path: static/mstile-150x150.png + key: MD5E-s4356--208ec7d8f732ec0cabe7b0a5c5c6287c.png + +committed_record_bundles: + datasets: 15 + instruments: 25 + objectives: 5 + persons: 27 + projects: 15 + publications: 846 + topics: 19 + +graph: + nodes: 1038 + edges: 2148 + nodes_with_urls: 998 + root_absolute_urls_before_pages_adapter: 998 + missing_page_targets: 26 + missing_page_targets_by_type: + organization: 8 + person: 8 + project: 10 + +static_build: + dump_things_processes: 0 + live_pool_requests: 0 + runs: 2 + hugo_pages: 1973 + non_page_files: 29 + static_files: 15 + processed_images: 34 + output_files: 2058 + unadapted_output_bytes: 102474096 + deterministic: true + exact_upstream_manifest_sha256: e4b7fac2a36ed5cb18e5a07c9cb6e2aef6bf5d82fb86df8c8d6e31285a240af7 + +github_pages_project_path_trial: + test_base_url: http://127.0.0.1:8766/orinoco-lite-dev/ + base_path: /orinoco-lite-dev/ + upstream_path_leaks: + html_references: 974 + graph_fetches: 1 + graph_node_urls: 998 + webmanifest_icon_urls: 2 + adapter: + html_files_changed: 962 + html_urls_rewritten: 974 + graph_script_urls_rewritten: 1 + graph_node_urls_rewritten: 998 + webmanifest_urls_rewritten: 2 + remaining_path_leaks: 0 + output_files: 2058 + output_bytes: 102852794 + repeat_manifest_sha256: a58fee0aec0d8725c72b7d26068dc340b742494c4520480f80555e1dc6246c14 + fresh_github_mirror_manifest_sha256: a58fee0aec0d8725c72b7d26068dc340b742494c4520480f80555e1dc6246c14 + browser: + homepage_graph_canvas_count: 7 + representative_term_graph_canvas_count: 7 + console_errors: 0 + console_warnings: 0 + +local_editor_bridge: + pool_ui_pin: 1572ef909321626f5efb2ea04b2f256e03ba50a4 + pool_ui_branch: codex/local-deployment + shacl_vue_pin: d5790a4431f7773a2e29fbb0d26e542ed0311ec5 + shacl_vue_url: http://127.0.0.1:3000/ + dump_things_service_pin: 9f101d97c7f15d491f602db5a9c33ad9a19ad8bf + dump_things_url: http://127.0.0.1:8111 + local_gitannex_url: http://127.0.0.1:8122/git-annex + source_public_snapshot_records: 4978 + local_public_curated_records: 4978 + local_protected_curated_records: 4978 + edit_urls_rewritten: 953 + browser_route_status: 200 + browser_selected_class: xyzri:XYZDataset + browser_record_pid_query: xyzrins:datasets/studyforrest + browser_form_opened: true + browser_submit_status: successful + local_service_backend: true + local_record_persistence: true + curation_semantics: incoming_overlay_until_curated + +known_upstream_snapshot_issues: + html_missing_targets: + unique: 7 + occurrences: 8 + graph_missing_targets: 26 + register_depictions_deposit_action_missing: true + +refresh_boundary: + live_api: https://pool.psychoinformatics.de/api + requires_dump_things_service_in_deploy: false + requires_live_pool_for_refresh: true + source_metadata_snapshot_committed: false + flow_reference: https://hub.psychoinformatics.de/orinoco/flow/forgejo/actions/prep-metadata-query@main + flow_commit_observed: 6971f35b8b2e2325aba6854e66360fed7424f6be + query_things_commit_observed: ef1141430a471455d4a5f4e07d7989ec717f56f4 + dump_things_pyclient_commit_observed: 1e79391195ad4412286344189dc5f81a06accb90 diff --git a/provenance/upstream-psychoinformatics/missing-targets.tsv b/provenance/upstream-psychoinformatics/missing-targets.tsv new file mode 100644 index 0000000..23440e4 --- /dev/null +++ b/provenance/upstream-psychoinformatics/missing-targets.tsv @@ -0,0 +1,35 @@ +surface type source_or_id target +html a.href instruments/8b75c829-479e-4bc4-8354-2fdf29a7d213/index.html /instruments/8b75c829-479e-4bc4-8354-2fdf29a7d213/hub.datalad.org/datalad/datalad-core +html a.href instruments/index.html /instruments/hub.datalad.org/datalad/datalad-core +html a.href projects/a605574d-ee49-4ad4-8348-599b632ed5eb/index.html /orcid:0000-0003-3456-2493 +html a.href projects/abcd-j/index.html /depictions/logo_abcd-j +html a.href projects/datalad/index.html /orcid:0000-0003-3456-2493 +html a.href projects/sfb1451/index.html /depictions/logo_sfb1451 +html a.href projects/studyforrest/index.html /depictions/141b8e85-5cc0-40f9-bf60-21cd89292177 +html a.href projects/trr379_q02/index.html /projects/trr379 +graph person orcid:0000-0003-3456-2493 /persons/yaroslav-halchenko +graph organization xyzrins:organizations/4985786a-5a4b-4b86-8fff-0b7be76a6227 /organizations/4985786a-5a4b-4b86-8fff-0b7be76a6227 +graph organization xyzrins:organizations/91f492bf-c0f7-4666-8eb0-5eb8d63070b8 /organizations/91f492bf-c0f7-4666-8eb0-5eb8d63070b8 +graph organization xyzrins:organizations/fzj /organizations/fzj +graph organization xyzrins:organizations/fzj-inm /organizations/fzj-inm +graph organization xyzrins:organizations/fzj-inm7 /organizations/fzj-inm7 +graph organization xyzrins:organizations/hhu /organizations/hhu +graph organization xyzrins:organizations/ovgu /organizations/ovgu +graph organization xyzrins:organizations/psyinf-group /organizations/psyinf-group +graph person xyzrins:persons/03cd9825-f9e4-43c4-baaf-1f077cdaf897 /persons/03cd9825-f9e4-43c4-baaf-1f077cdaf897 +graph person xyzrins:persons/0d5d403c-9ae7-4e81-9a73-ce233a56ff27 /persons/0d5d403c-9ae7-4e81-9a73-ce233a56ff27 +graph person xyzrins:persons/905e4666-3239-4dac-a3bd-8bdc18f70714 /persons/905e4666-3239-4dac-a3bd-8bdc18f70714 +graph person xyzrins:persons/a61dbb70-d0b0-4bb1-8888-01242d22e4cd /persons/a61dbb70-d0b0-4bb1-8888-01242d22e4cd +graph person xyzrins:persons/cd0bc74d-9849-4891-a852-84e53736f505 /persons/cd0bc74d-9849-4891-a852-84e53736f505 +graph person xyzrins:persons/f15451c2-7985-4c85-9426-a347aa818753 /persons/f15451c2-7985-4c85-9426-a347aa818753 +graph person xyzrins:persons/fe4c3a56-8607-4806-abfe-18312f3ebde7 /persons/fe4c3a56-8607-4806-abfe-18312f3ebde7 +graph project xyzrins:projects/05f030d0-f9e5-4f85-aa5e-ccd4104856a1 /projects/05f030d0-f9e5-4f85-aa5e-ccd4104856a1 +graph project xyzrins:projects/236dde03-0176-4747-ad0f-c6346034d05e /projects/236dde03-0176-4747-ad0f-c6346034d05e +graph project xyzrins:projects/45e542dd-2d2a-41d0-8399-90f2fde72a21 /projects/45e542dd-2d2a-41d0-8399-90f2fde72a21 +graph project xyzrins:projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725 /projects/5340dd43-6ef0-4cf8-a936-67c2a9f3a725 +graph project xyzrins:projects/64777d99-26f6-441c-9849-ba8018571de8 /projects/64777d99-26f6-441c-9849-ba8018571de8 +graph project xyzrins:projects/7d475f77-48e7-4d67-b121-274b4582e561 /projects/7d475f77-48e7-4d67-b121-274b4582e561 +graph project xyzrins:projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5 /projects/858d0ddb-3c51-4cea-a0e5-a73224b59ae5 +graph project xyzrins:projects/9c5e518c-c25c-4bc6-a85f-ade51f8308cd /projects/9c5e518c-c25c-4bc6-a85f-ade51f8308cd +graph project xyzrins:projects/a3f0a9e0-c945-4e04-a698-be426a9ac075 /projects/a3f0a9e0-c945-4e04-a698-be426a9ac075 +graph project xyzrins:projects/trr379 /projects/trr379 diff --git a/provenance/upstream-psychoinformatics/submodule-inventory.tsv b/provenance/upstream-psychoinformatics/submodule-inventory.tsv new file mode 100644 index 0000000..f8a0f07 --- /dev/null +++ b/provenance/upstream-psychoinformatics/submodule-inventory.tsv @@ -0,0 +1,25 @@ +path parent_main_pin configured_branch default_branch current_default_head status workflow_role con_repository +submodules/artwork 890a92fe62d6de0e142724fe1e79a4e82b920c4c main main 890a92fe62d6de0e142724fe1e79a4e82b920c4c exact irrelevant none +submodules/bids-things d0d4dfd9fc40f0e05daac28b8c3b151c25efb76b main main d0d4dfd9fc40f0e05daac28b8c3b151c25efb76b exact irrelevant none +submodules/centerforopenneuroscience.org 578723864c7ee55c4ec541fd41e5cf82055b434f orinoco-lite orinoco-lite 578723864c7ee55c4ec541fd41e5cf82055b434f exact account mirror; nested congo URL points to leej3/congo irrelevant leej3/centerforopenneuroscience.org +submodules/curatee-kit de3d023683f62419aa4a1d70b4dfad32b971beb5 main main de3d023683f62419aa4a1d70b4dfad32b971beb5 exact irrelevant none +submodules/dump-research-info 1c7e99ec6f296d5e6cb6a61e3b786227190802da agent/git-native-con-metadata (missing) main 60e8fb79c11eae62eaf17637d4d79f3bbc7edb98 divergent: pin-only 20, default-only 4 irrelevant native con repository +submodules/dump-things-pyclient 1e79391195ad4412286344189dc5f81a06accb90 master master 1e79391195ad4412286344189dc5f81a06accb90 exact metadata refresh none +submodules/dump-things-service 9f101d97c7f15d491f602db5a9c33ad9a19ad8bf master master 9f101d97c7f15d491f602db5a9c33ad9a19ad8bf exact live backend only none +submodules/dump-things-service-mirror 0c883522f933bc89f1ec87c4ab7848015acce80f master master 0c883522f933bc89f1ec87c4ab7848015acce80f exact irrelevant none +submodules/find-things 050d68ca9c5b9d8d543cfae20ee1486e90d5a6cf main main 050d68ca9c5b9d8d543cfae20ee1486e90d5a6cf exact irrelevant none +submodules/flatson e25086253b76a133471a7674f081547459087415 main main e25086253b76a133471a7674f081547459087415 exact irrelevant none +submodules/flatson-js 2e2763ee5b1c1caef7173b1cae8f3cbae3ba765b main main 2e2763ee5b1c1caef7173b1cae8f3cbae3ba765b exact irrelevant none +submodules/flatsonpy 5a26ee71de35e30ae66ebd81c64e2249367bb0b4 main main 5a26ee71de35e30ae66ebd81c64e2249367bb0b4 exact irrelevant none +submodules/flow 6971f35b8b2e2325aba6854e66360fed7424f6be main main 6971f35b8b2e2325aba6854e66360fed7424f6be exact metadata refresh none +submodules/psyinf-pool-files-public d5689b11c67d5232f84782c73ed003e847734557 main main d5689b11c67d5232f84782c73ed003e847734557 exact irrelevant none +submodules/query-things ef1141430a471455d4a5f4e07d7989ec717f56f4 main main ef1141430a471455d4a5f4e07d7989ec717f56f4 exact metadata refresh none +submodules/research-information-ui-assets f1b744e5d009c61b77bd3ddd6fde29c2ad46ea69 main main f1b744e5d009c61b77bd3ddd6fde29c2ad46ea69 exact irrelevant none +submodules/shacl-tulip fee25fdfc00c4898e4a928814f576c027c75d678 main main fee25fdfc00c4898e4a928814f576c027c75d678 exact irrelevant none +submodules/pool.psychoinformatics.de-ui 1572ef909321626f5efb2ea04b2f256e03ba50a4 codex/local-deployment main 45a698f58204c5413c757fe44ed47cd04e504674 deployment branch tracks local Dump Things, schema assets, git-annex configuration, and nested account URL local editor and local service stack leej3/pool.psychoinformatics.de-ui +submodules/some-things ee1dcdac44076d94a70dcd3c2f07f404b4e69c14 main main ee1dcdac44076d94a70dcd3c2f07f404b4e69c14 exact irrelevant none +submodules/things-enrichment-tools d15d0866e3208eb21365dac978d747aedb63b456 main main dcfcceda15987ce51ebf51ce1583824d474639fc behind by 6 commits irrelevant none +submodules/things-graph-renderer 04f6241e37532fdb03b6f95d2dbe304e7171d504 main main 04f6241e37532fdb03b6f95d2dbe304e7171d504 exact maintenance only none +submodules/things-schemas d26ea4135e28c25b134c64de1cdc15d15cd2f9f0 main main d26ea4135e28c25b134c64de1cdc15d15cd2f9f0 exact irrelevant to current workflow none +submodules/tools c76e9a9c547acddab3efd510c73e341793ca5cfa main main c76e9a9c547acddab3efd510c73e341793ca5cfa exact account mirror; nested datalad-concepts URL points to leej3/datalad-concepts irrelevant leej3/tools +submodules/www-from-model 6c8b9a5b7260dc20dfe1453dd863b353e8f90f06 main main 6c8b9a5b7260dc20dfe1453dd863b353e8f90f06 exact account mirror; nested congo URL points to leej3/congo static build and metadata refresh leej3/www-from-model diff --git a/submodules/centerforopenneuroscience.org b/submodules/centerforopenneuroscience.org index 2621231..26907c4 160000 --- a/submodules/centerforopenneuroscience.org +++ b/submodules/centerforopenneuroscience.org @@ -1 +1 @@ -Subproject commit 2621231d27b70fb425107a132159f7a9e0d99cda +Subproject commit 26907c487efaa2c31bba9d02398aa201ab6f774b diff --git a/submodules/dump-research-info b/submodules/dump-research-info index 1c7e99e..062da59 160000 --- a/submodules/dump-research-info +++ b/submodules/dump-research-info @@ -1 +1 @@ -Subproject commit 1c7e99ec6f296d5e6cb6a61e3b786227190802da +Subproject commit 062da59cb5a00ca128b3df895426a54088bfc625 diff --git a/submodules/pool.psychoinformatics.de-ui b/submodules/pool.psychoinformatics.de-ui new file mode 160000 index 0000000..93961ac --- /dev/null +++ b/submodules/pool.psychoinformatics.de-ui @@ -0,0 +1 @@ +Subproject commit 93961ace8d4ceaea088ccc04526a9bc5428139a6 diff --git a/submodules/shacl-vue b/submodules/shacl-vue deleted file mode 160000 index dbb9bfa..0000000 --- a/submodules/shacl-vue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dbb9bfa3997881abb0dfc8ba1bbc572ddc39b8d0 diff --git a/submodules/tools b/submodules/tools index 729f418..c76e9a9 160000 --- a/submodules/tools +++ b/submodules/tools @@ -1 +1 @@ -Subproject commit 729f4186c08728e3354412f650d961e9702743df +Subproject commit c76e9a9c547acddab3efd510c73e341793ca5cfa diff --git a/submodules/www-from-model b/submodules/www-from-model index 6945272..6c8b9a5 160000 --- a/submodules/www-from-model +++ b/submodules/www-from-model @@ -1 +1 @@ -Subproject commit 6945272e5f3fcf353627b8e1c3e68bcaf76cc2ce +Subproject commit 6c8b9a5b7260dc20dfe1453dd863b353e8f90f06 diff --git a/tests/browser/authenticated-editor.spec.mjs b/tests/browser/authenticated-editor.spec.mjs new file mode 100644 index 0000000..1466800 --- /dev/null +++ b/tests/browser/authenticated-editor.spec.mjs @@ -0,0 +1,95 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { expect, test } from '@playwright/test'; + +import { + cleanupProbe, + loadEditorToken, + loadProbeRecord, + PROBE_PID, + readProbeBoundaries, + seedProbe, +} from './dump-things.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const PERSON_URL = 'http://127.0.0.1:8767/persons/yaroslav-halchenko/'; + +test.use({ trace: 'off', screenshot: 'off', video: 'off' }); + +test('an authenticated SHACL Vue edit reaches only the CON incoming boundary', async ({ + context, + page, +}) => { + const probe = await loadProbeRecord(ROOT); + await cleanupProbe(ROOT); + try { + await seedProbe(ROOT, probe); + await page.goto(PERSON_URL); + const originalHref = await page + .getByRole('link', { name: 'Edit this record' }) + .getAttribute('href'); + const editorURL = new URL(originalHref); + editorURL.searchParams.set('pid', PROBE_PID); + + const editorToken = await loadEditorToken(ROOT); + const editor = await context.newPage(); + await editor.addInitScript((token) => { + sessionStorage.setItem('serviceToken', token); + }, editorToken); + + let authenticatedPost; + editor.on('request', async (request) => { + const url = new URL(request.url()); + if ( + request.method() === 'POST' + && url.pathname === '/con-protected/record/XYZPerson' + ) { + authenticatedPost = request; + } + }); + await editor.goto(editorURL.href); + await expect(editor.getByText('Person', { exact: true }).first()).toBeVisible(); + + const givenNameRow = editor.locator('.main-row').filter({ + has: editor.locator('.row-label', { hasText: /Given name/i }), + }); + const givenName = givenNameRow.locator('input').first(); + await expect(givenName).toHaveValue('Playwright'); + await givenName.fill('Browser-tested'); + await editor.getByRole('button', { name: 'Save', exact: true }).click(); + + await editor.locator('button:has(.mdi-cloud-upload)').click(); + const submission = editor.locator('#submitcomp'); + await expect( + submission.getByRole('checkbox', { name: /Playwright Write Probe/ }), + ).toBeChecked(); + await submission.getByRole('button', { name: 'Submit', exact: true }).click(); + await expect( + submission.getByText('Your metadata submission was successful!', { exact: true }), + ).toBeVisible(); + + expect(authenticatedPost).toBeDefined(); + const postURL = new URL(authenticatedPost.url()); + expect(postURL.pathname).toBe('/con-protected/record/XYZPerson'); + expect(postURL.searchParams.get('format')).toBe('ttl'); + const postHeaders = await authenticatedPost.allHeaders(); + expect(Boolean(postHeaders['x-dumpthings-token'])).toBe(true); + expect(postHeaders['content-type']).toContain('text/turtle'); + + const boundaries = await readProbeBoundaries(ROOT); + expect(boundaries['con-protected'].curated.given_name).toBe('Playwright'); + expect(boundaries['con-protected']['incoming/local-editor'].given_name).toBe( + 'Browser-tested', + ); + for (const [collection, areas] of Object.entries(boundaries)) { + if (collection === 'con-protected') { + continue; + } + expect(areas.curated).toBeNull(); + expect(areas['incoming/local-editor']).toBeNull(); + } + } finally { + await cleanupProbe(ROOT); + } +}); diff --git a/tests/browser/dump-things.mjs b/tests/browser/dump-things.mjs new file mode 100644 index 0000000..4db6400 --- /dev/null +++ b/tests/browser/dump-things.mjs @@ -0,0 +1,106 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +export const SERVICE_URL = 'http://127.0.0.1:8111'; +export const COLLECTIONS = [ + 'upstream-public', + 'upstream-protected', + 'con-public', + 'con-protected', +]; +export const PROBE_PID = 'xyzrins:persons/_clean-migration-playwright-write-probe'; +export const PROBE_CLASS = 'XYZPerson'; + +async function secret(root, name) { + return (await readFile(path.join(root, 'build/local-stack', name), 'utf8')).trim(); +} + +async function request(url, { method = 'GET', token, body, allowMissing = false } = {}) { + const headers = { Accept: 'application/json' }; + if (token !== undefined) { + headers['X-DumpThings-Token'] = token; + } + let payload; + if (body !== undefined) { + headers['Content-Type'] = 'application/json'; + payload = JSON.stringify(body); + } + const response = await fetch(url, { method, headers, body: payload }); + if (allowMissing && response.status === 404) { + return null; + } + if (!response.ok) { + throw new Error(`${method} ${new URL(url).pathname} failed with ${response.status}`); + } + const text = await response.text(); + return text ? JSON.parse(text) : null; +} + +function recordURL(collection, boundary, pid) { + const query = new URLSearchParams({ pid }); + return `${SERVICE_URL}/${collection}/${boundary}/record?${query}`; +} + +export async function loadEditorToken(root) { + return secret(root, 'editor-token'); +} + +export async function loadProbeRecord(root) { + const source = await readFile(path.join(root, 'build/con-projection/records.jsonl'), 'utf8'); + const envelope = source + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((item) => item.record?.pid === 'xyzrins:persons/yaroslav-halchenko'); + if (envelope?.class_name !== PROBE_CLASS) { + throw new Error('The canonical Yaroslav test fixture is unavailable'); + } + const record = structuredClone(envelope.record); + record.pid = PROBE_PID; + record.display_label = 'Playwright Write Probe'; + record.formatted_name = 'Playwright Write Probe'; + record.given_name = 'Playwright'; + record.family_name = 'Probe'; + record.additional_names = ['Browser']; + record.identifiers = [ + { notation: 'clean-migration-playwright-probe', schema_type: 'dlthings:Identifier' }, + ]; + return record; +} + +export async function cleanupProbe(root) { + const token = await secret(root, 'seed-token'); + for (const collection of COLLECTIONS) { + for (const boundary of ['curated', 'incoming/local-editor']) { + await request(recordURL(collection, boundary, PROBE_PID), { + method: 'DELETE', + token, + allowMissing: true, + }); + } + } +} + +export async function seedProbe(root, record) { + const token = await secret(root, 'seed-token'); + await request(`${SERVICE_URL}/con-protected/curated/record/${PROBE_CLASS}`, { + method: 'POST', + token, + body: record, + }); +} + +export async function readProbeBoundaries(root) { + const token = await secret(root, 'seed-token'); + const result = {}; + for (const collection of COLLECTIONS) { + result[collection] = {}; + for (const boundary of ['curated', 'incoming/local-editor']) { + result[collection][boundary] = await request( + recordURL(collection, boundary, PROBE_PID), + { token, allowMissing: true }, + ); + } + } + return result; +} diff --git a/tests/browser/graph-cache.spec.mjs b/tests/browser/graph-cache.spec.mjs new file mode 100644 index 0000000..1240637 --- /dev/null +++ b/tests/browser/graph-cache.spec.mjs @@ -0,0 +1,127 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { expect, test } from '@playwright/test'; + +import { startStaticServer } from './static-server.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const CON_SITE_ROOT = process.env.CON_SITE_ROOT ?? path.join( + ROOT, + 'submodules/centerforopenneuroscience.org', +); +const COMMITTED_GRAPH = path.join( + CON_SITE_ROOT, + 'profiles/con/projection/static/graph.json', +); +const REPRESENTATIVE_CON_PIDS = [ + 'xyzrins:persons/yaroslav-halchenko', + 'xyzrins:projects/datalad', +]; + +async function expectedCONGraph() { + return JSON.parse(await readFile(COMMITTED_GRAPH, 'utf8')); +} + +function edgePairs(graph) { + return graph.edges.map((edge) => `${edge.source}\0${edge.target}`).sort(); +} + +function graphResponse(page) { + return page.waitForResponse((response) => { + const url = new URL(response.url()); + return url.pathname.endsWith('/graph.json'); + }); +} + +test( + 'same-origin upstream cache cannot contaminate the CON graph', + async ({ page }) => { + const fixture = await startStaticServer( + { + upstream: path.join(ROOT, 'build/upstream-local'), + con: path.join(ROOT, 'build/con-site'), + }, + 'upstream', + ); + try { + const expected = await expectedCONGraph(); + const firstGraph = graphResponse(page); + await page.goto(`${fixture.origin}/`); + const upstreamResponse = await firstGraph; + const upstreamURL = new URL(upstreamResponse.url()); + const upstream = await upstreamResponse.json(); + expect(upstreamURL.searchParams.get('v')).toMatch(/^[0-9a-f]{64}$/); + expect(upstream.nodes.length).toBeGreaterThan(100); + expect(upstream.nodes.some((node) => node.label === 'PsyInf')).toBe(true); + await expect(page.locator('#sigma-container canvas').first()).toBeVisible(); + + fixture.use('con'); + const secondGraph = graphResponse(page); + await page.reload(); + const conResponse = await secondGraph; + const conURL = new URL(conResponse.url()); + const con = await conResponse.json(); + expect(conURL.searchParams.get('v')).toMatch(/^[0-9a-f]{64}$/); + expect(conURL.href).not.toBe(upstreamURL.href); + expect(new Set(con.nodes.map((node) => node.id))).toEqual( + new Set(expected.nodes.map((node) => node.id)), + ); + expect(edgePairs(con)).toEqual(edgePairs(expected)); + expect(con.edges).toHaveLength(expected.edges.length); + for (const pid of REPRESENTATIVE_CON_PIDS) { + expect(con.nodes.some((node) => node.id === pid)).toBe(true); + } + expect(con.nodes.map((node) => node.label)).not.toEqual( + expect.arrayContaining(['PsyInf', 'FZJ', 'M.Hanke']), + ); + await expect( + page.getByRole('heading', { name: 'Center for Open Neuroscience' }), + ).toBeVisible(); + await expect(page.locator('#sigma-container canvas').first()).toBeVisible(); + + const scriptSource = await page + .locator('script[src*="graph.js"]') + .first() + .getAttribute('src'); + expect(new URL(scriptSource, fixture.origin).searchParams.get('v')).toBe( + conURL.searchParams.get('v'), + ); + } finally { + await fixture.close(); + } + }, +); + +test('project-path graph resources and routes resolve', async ({ page }) => { + const fixture = await startStaticServer( + { con: path.join(ROOT, 'build/con-site-project') }, + 'con', + { mountPath: '/full-con-migration/' }, + ); + try { + const graph = graphResponse(page); + await page.goto(`${fixture.origin}/full-con-migration/`); + const response = await graph; + expect(new URL(response.url()).pathname).toBe('/full-con-migration/graph.json'); + expect(new URL(response.url()).searchParams.get('v')).toMatch( + /^[0-9a-f]{64}$/, + ); + const personHref = await page + .getByRole('link', { name: 'Yaroslav Halchenko' }) + .first() + .getAttribute('href'); + const personPath = new URL(personHref).pathname; + expect(personPath).toBe('/full-con-migration/persons/yaroslav-halchenko/'); + await page.goto(`${fixture.origin}${personPath}`); + await expect(page).toHaveURL( + /\/full-con-migration\/persons\/yaroslav-halchenko\/$/, + ); + await expect( + page.getByRole('heading', { name: 'Yaroslav Halchenko' }), + ).toBeVisible(); + } finally { + await fixture.close(); + } +}); diff --git a/tests/browser/shacl-editor.spec.mjs b/tests/browser/shacl-editor.spec.mjs new file mode 100644 index 0000000..4d4d5fd --- /dev/null +++ b/tests/browser/shacl-editor.spec.mjs @@ -0,0 +1,58 @@ +import { expect, test } from '@playwright/test'; + +const PERSON_URL = 'http://127.0.0.1:8767/persons/yaroslav-halchenko/'; +const PERSON_PID = 'xyzrins:persons/yaroslav-halchenko'; + +function isPersonRecordResponse(response) { + const url = new URL(response.url()); + return ( + url.origin === 'http://127.0.0.1:8111' + && url.pathname === '/con-protected/record' + && url.searchParams.get('pid') === PERSON_PID + ); +} + +test('the real Yaroslav edit link opens a populated anonymous Person form', async ({ + context, + page, +}) => { + const recordResponses = []; + context.on('response', (response) => { + if (isPersonRecordResponse(response)) { + recordResponses.push(response); + } + }); + + await page.goto(PERSON_URL); + const editLink = page.getByRole('link', { name: 'Edit this record' }); + const href = await editLink.getAttribute('href'); + const editURL = new URL(href); + expect(editURL.origin).toBe('http://127.0.0.1:3000'); + expect([...editURL.searchParams.keys()].sort()).toEqual([ + 'edit', + 'pid', + 'sh:NodeShape', + ]); + expect(editURL.searchParams.get('sh:NodeShape')).toBe('dlthings:Thing'); + expect(editURL.searchParams.get('pid')).toBe(PERSON_PID); + expect(editURL.searchParams.get('edit')).toBe('true'); + + const popupPromise = context.waitForEvent('page'); + await editLink.click(); + const editor = await popupPromise; + await editor.waitForLoadState('domcontentloaded'); + await expect.poll(() => recordResponses.length).toBeGreaterThan(0); + expect(recordResponses.at(-1).status()).toBe(200); + const requestHeaders = await recordResponses.at(-1).request().allHeaders(); + expect(requestHeaders).not.toHaveProperty('x-dumpthings-token'); + + await expect(editor.getByText('Person', { exact: true }).first()).toBeVisible(); + await expect + .poll(async () => + editor + .locator('input') + .evaluateAll((inputs) => inputs.map((item) => item.value)), + ) + .toEqual(expect.arrayContaining(['Yaroslav', 'Halchenko'])); + await expect(editor.getByText('No items', { exact: true })).toHaveCount(0); +}); diff --git a/tests/browser/static-editor.spec.mjs b/tests/browser/static-editor.spec.mjs new file mode 100644 index 0000000..0836e3f --- /dev/null +++ b/tests/browser/static-editor.spec.mjs @@ -0,0 +1,103 @@ +import { execFile } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +import { expect, test } from '@playwright/test'; + +import { startStaticServer } from './static-server.mjs'; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const PAGES_ROOT = path.join( + ROOT, + 'build/pages-preview', +); +const PERSON_PID = 'xyzrins:persons/yaroslav-halchenko'; +const execFileAsync = promisify(execFile); + +test('project-path editor changes a record and downloads without a backend write', async ({ + context, + page, +}) => { + await context.addInitScript(() => { + sessionStorage.setItem('serviceToken', 'inherited-static-token'); + }); + const fixture = await startStaticServer({ pages: PAGES_ROOT }, 'pages'); + const mutationRequests = []; + context.on('request', (request) => { + if (!['GET', 'HEAD'].includes(request.method())) mutationRequests.push(request); + }); + try { + await page.goto(`${fixture.origin}/orinoco-lite-dev/persons/yaroslav-halchenko/`); + const publishedEdit = new URL( + await page.getByRole('link', { name: 'Edit this record' }).getAttribute('href'), + ); + const editorURL = new URL('/orinoco-lite-dev/edit/', fixture.origin); + editorURL.search = publishedEdit.search; + editorURL.searchParams.set('token', 'query-static-token'); + await page.goto(editorURL.href); + + await expect(page.getByText('Person', { exact: true }).first()).toBeVisible(); + expect(new URL(page.url()).searchParams.has('token')).toBe(false); + const givenNameRow = page.locator('.main-row').filter({ + has: page.locator('.row-label', { hasText: /Given name/i }), + }); + const givenName = givenNameRow.locator('input').first(); + await expect(givenName).toHaveValue('Yaroslav'); + await givenName.fill('Yaroslav Browser Review'); + await page.getByRole('button', { name: 'Save', exact: true }).click(); + + await page.locator('button:has(.mdi-download)').first().click(); + const submission = page.locator('#submitcomp'); + await expect( + submission.getByRole('checkbox', { name: /Yaroslav/ }), + ).toBeChecked(); + const downloadPromise = page.waitForEvent('download'); + await submission + .getByRole('button', { name: 'Download review bundle', exact: true }) + .click(); + const download = await downloadPromise; + expect(download.suggestedFilename()).toBe( + 'con-review-xyzrins-persons-yaroslav-halchenko.json', + ); + const downloaded = await download.path(); + const bundle = JSON.parse(await readFile(downloaded, 'utf8')); + expect(bundle.format).toBe('con-shacl-review-bundle'); + expect(bundle.version).toBe(1); + expect(bundle.site_commit).toMatch(/^[0-9a-f]{40}$/); + expect(bundle.records).toHaveLength(1); + expect(bundle.records[0]).toMatchObject({ + pid: PERSON_PID, + schema_type: 'xyzri:XYZPerson', + source_path: + 'profiles/con/metadata/records/XYZPerson/yaroslav-halchenko.yaml', + }); + expect(bundle.records[0].source_sha256).toMatch(/^[0-9a-f]{64}$/); + expect(bundle.records[0].rdf_turtle).toContain('Yaroslav Browser Review'); + + const dryRun = await execFileAsync( + 'python3', + [path.join(ROOT, 'tools/apply_editor_bundle.py'), downloaded], + { cwd: ROOT, maxBuffer: 10 * 1024 * 1024 }, + ); + expect(dryRun.stdout).toContain( + 'b/profiles/con/metadata/records/XYZPerson/yaroslav-halchenko.yaml', + ); + expect(dryRun.stdout).toContain('Yaroslav Browser Review'); + expect(dryRun.stdout).toContain('Dry run only'); + + expect(mutationRequests).toEqual([]); + expect(fixture.requests.every(({ method }) => ['GET', 'HEAD'].includes(method))).toBe( + true, + ); + expect( + await page.evaluate(() => ({ + local: Object.keys(localStorage), + session: Object.keys(sessionStorage), + })), + ).toEqual({ local: [], session: [] }); + } finally { + await fixture.close(); + } +}); diff --git a/tests/browser/static-server.mjs b/tests/browser/static-server.mjs new file mode 100644 index 0000000..830567d --- /dev/null +++ b/tests/browser/static-server.mjs @@ -0,0 +1,106 @@ +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import path from 'node:path'; + +const CONTENT_TYPES = new Map([ + ['.css', 'text/css; charset=utf-8'], + ['.html', 'text/html; charset=utf-8'], + ['.js', 'text/javascript; charset=utf-8'], + ['.json', 'application/json; charset=utf-8'], + ['.png', 'image/png'], + ['.svg', 'image/svg+xml'], + ['.ttl', 'text/turtle; charset=utf-8'], + ['.webmanifest', 'application/manifest+json'], +]); + +function normalizeMountPath(value) { + const withLeading = value.startsWith('/') ? value : `/${value}`; + return withLeading.endsWith('/') ? withLeading : `${withLeading}/`; +} + +function resolveRequest(root, requestPath, mountPath) { + if (!requestPath.startsWith(mountPath)) { + return null; + } + let relative = decodeURIComponent(requestPath.slice(mountPath.length)); + if (!relative || relative.endsWith('/')) { + relative += 'index.html'; + } + const candidate = path.resolve(root, relative); + const resolvedRoot = path.resolve(root); + if (candidate !== resolvedRoot && !candidate.startsWith(`${resolvedRoot}${path.sep}`)) { + return null; + } + return candidate; +} + +export async function startStaticServer(profiles, initialProfile, options = {}) { + let activeProfile = initialProfile; + const mountPath = normalizeMountPath(options.mountPath ?? '/'); + const requests = []; + const server = createServer(async (request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + const root = profiles[activeProfile]; + const file = resolveRequest(root, url.pathname, mountPath); + requests.push({ + method: request.method, + profile: activeProfile, + path: url.pathname, + search: url.search, + }); + if (file === null) { + response.writeHead(404).end('Not found'); + return; + } + try { + const info = await stat(file); + if (!info.isFile()) { + response.writeHead(404).end('Not found'); + return; + } + const extension = path.extname(file); + response.setHeader( + 'Content-Type', + CONTENT_TYPES.get(extension) ?? 'application/octet-stream', + ); + if (path.basename(file) === 'graph.js' || path.basename(file) === 'graph.json') { + response.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); + } else if (extension === '.html') { + response.setHeader('Cache-Control', 'no-store'); + } + response.setHeader('Content-Length', info.size); + createReadStream(file).pipe(response); + } catch (error) { + if (error?.code === 'ENOENT') { + response.writeHead(404).end('Not found'); + return; + } + response.writeHead(500).end('Static fixture failure'); + } + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + if (address === null || typeof address === 'string') { + throw new Error('Could not determine static fixture address'); + } + return { + origin: `http://127.0.0.1:${address.port}`, + mountPath, + requests, + use(profile) { + if (!(profile in profiles)) { + throw new Error(`Unknown static fixture profile: ${profile}`); + } + activeProfile = profile; + }, + async close() { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} diff --git a/tests/test_adapt_upstream_pages.py b/tests/test_adapt_upstream_pages.py new file mode 100644 index 0000000..ec4aff1 --- /dev/null +++ b/tests/test_adapt_upstream_pages.py @@ -0,0 +1,325 @@ +import json +import re +import tempfile +import unittest +from pathlib import Path + +from tools.adapt_upstream_pages import ( + adapt_site, + audit_site, + normalize_base_path, + prefix_root_url, +) + + +VERSION_RE = re.compile(r"\?v=([0-9a-f]{64})") + + +def write_graph_site(site_dir: Path, *, label: str = "Example") -> None: + (site_dir / "index.html").write_text( + '', encoding="utf-8" + ) + (site_dir / "graph.js").write_text( + 'async function load(){return fetch("/graph.json")}', encoding="utf-8" + ) + (site_dir / "graph.json").write_text( + json.dumps( + { + "nodes": [ + {"id": "example", "label": label, "url": "/persons/example"} + ], + "edges": [], + } + ), + encoding="utf-8", + ) + + +def graph_version(site_dir: Path) -> str: + match = VERSION_RE.search((site_dir / "index.html").read_text(encoding="utf-8")) + if match is None: + raise AssertionError("site index has no graph bundle version") + return match.group(1) + + +class AdaptUpstreamPagesTests(unittest.TestCase): + def test_normalize_base_path(self) -> None: + self.assertEqual(normalize_base_path("/"), "/") + self.assertEqual(normalize_base_path("/orinoco-lite-dev"), "/orinoco-lite-dev/") + self.assertEqual( + normalize_base_path("/orinoco-lite-dev/"), "/orinoco-lite-dev/" + ) + with self.assertRaises(ValueError): + normalize_base_path("orinoco-lite-dev") + with self.assertRaises(ValueError): + normalize_base_path("/../escape") + + def test_prefix_root_url_is_idempotent(self) -> None: + base = "/orinoco-lite-dev/" + self.assertEqual(prefix_root_url("/graph.js", base), f"{base}graph.js") + self.assertEqual(prefix_root_url(f"{base}graph.js", base), f"{base}graph.js") + self.assertEqual( + prefix_root_url("//example.test/a.js", base), "//example.test/a.js" + ) + self.assertEqual( + prefix_root_url("https://example.test/a.js", base), + "https://example.test/a.js", + ) + + def test_audit_normalizes_base_path(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + (site_dir / "index.html").write_text( + '', encoding="utf-8" + ) + violations = audit_site(site_dir, "/orinoco-lite-dev") + self.assertIn("index.html: /graph.js", violations) + self.assertIn( + "index.html: graph script URL /graph.js has no complete graph bundle", + violations, + ) + + def test_root_base_path_requires_and_adds_graph_versions(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + write_graph_site(site_dir) + + before = audit_site(site_dir, "/") + self.assertTrue( + any("graph script URL /graph.js" in violation for violation in before) + ) + self.assertTrue( + any("graph data URL /graph.json" in violation for violation in before) + ) + + stats = adapt_site(site_dir, "/") + key = graph_version(site_dir) + self.assertEqual(len(key), 64) + self.assertEqual(stats.graph_html_urls_versioned, 1) + self.assertEqual(stats.graph_script_urls_rewritten, 1) + self.assertIn( + f'fetch("/graph.json?v={key}")', + (site_dir / "graph.js").read_text(encoding="utf-8"), + ) + self.assertEqual(audit_site(site_dir, "/"), []) + + def test_edit_url_rewrite_is_configurable_and_idempotent(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + (site_dir / "index.html").write_text( + 'Edit', + encoding="utf-8", + ) + + local_edit_url = "http://127.0.0.1:3000/" + before = audit_site(site_dir, "/", local_edit_url) + self.assertEqual( + before, ["index.html: edit URL https://pool.psychoinformatics.de/ui/"] + ) + + stats = adapt_site(site_dir, "/", local_edit_url) + self.assertEqual(stats.edit_urls_rewritten, 1) + self.assertIn( + 'href="http://127.0.0.1:3000/?pid=example&edit=true"', + (site_dir / "index.html").read_text(encoding="utf-8"), + ) + self.assertEqual(audit_site(site_dir, "/", local_edit_url), []) + self.assertEqual( + adapt_site(site_dir, "/", local_edit_url).edit_urls_rewritten, 0 + ) + + def test_adapt_site_rewrites_all_upstream_escape_types(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + (site_dir / "nested").mkdir() + (site_dir / "index.html").write_text( + 'Explore', + encoding="utf-8", + ) + (site_dir / "nested" / "index.html").write_text( + "Project", + encoding="utf-8", + ) + (site_dir / "graph.js").write_text( + 'async function load(){return fetch("/graph.json")}', + encoding="utf-8", + ) + (site_dir / "graph.json").write_text( + json.dumps( + { + "nodes": [ + {"id": "person", "url": "/persons/example"}, + {"id": "external", "url": "https://example.test/person"}, + {"id": "none", "url": None}, + ], + "edges": [], + } + ), + encoding="utf-8", + ) + (site_dir / "site.webmanifest").write_text( + json.dumps( + { + "icons": [ + {"src": "/android-chrome-192x192.png"}, + {"src": "https://example.test/external.png"}, + ] + } + ), + encoding="utf-8", + ) + + before = audit_site(site_dir, "/orinoco-lite-dev") + self.assertGreaterEqual(len(before), 7) + + stats = adapt_site(site_dir, "/orinoco-lite-dev") + key = graph_version(site_dir) + self.assertEqual(stats.html_files_changed, 2) + self.assertEqual(stats.html_urls_rewritten, 4) + self.assertEqual(stats.graph_html_urls_versioned, 1) + self.assertEqual(stats.graph_script_urls_rewritten, 1) + self.assertEqual(stats.graph_node_urls_rewritten, 1) + self.assertEqual(stats.webmanifest_urls_rewritten, 1) + self.assertIn( + f'src=/orinoco-lite-dev/graph.js?v={key}', + (site_dir / "index.html").read_text(encoding="utf-8"), + ) + self.assertIn( + f'fetch("/orinoco-lite-dev/graph.json?v={key}")', + (site_dir / "graph.js").read_text(encoding="utf-8"), + ) + self.assertEqual(audit_site(site_dir, "/orinoco-lite-dev"), []) + + # A second pass proves that deployment retries are deterministic. + before_second = { + path.relative_to(site_dir): path.read_bytes() + for path in site_dir.rglob("*") + if path.is_file() + } + second = adapt_site(site_dir, "/orinoco-lite-dev") + self.assertEqual(second.html_urls_rewritten, 0) + self.assertEqual(second.graph_html_urls_versioned, 0) + self.assertEqual(second.graph_script_urls_rewritten, 0) + self.assertEqual(second.graph_node_urls_rewritten, 0) + self.assertEqual(second.webmanifest_urls_rewritten, 0) + self.assertEqual( + before_second, + { + path.relative_to(site_dir): path.read_bytes() + for path in site_dir.rglob("*") + if path.is_file() + }, + ) + + def test_root_and_project_paths_have_distinct_bundle_keys(self) -> None: + with ( + tempfile.TemporaryDirectory() as root_temp, + tempfile.TemporaryDirectory() as project_temp, + ): + root_site = Path(root_temp) + project_site = Path(project_temp) + write_graph_site(root_site) + write_graph_site(project_site) + + adapt_site(root_site, "/") + adapt_site(project_site, "/clean-migration/") + + self.assertNotEqual(graph_version(root_site), graph_version(project_site)) + self.assertEqual(audit_site(root_site, "/"), []) + self.assertEqual(audit_site(project_site, "/clean-migration/"), []) + + def test_distinct_graph_bytes_produce_distinct_resource_urls(self) -> None: + with ( + tempfile.TemporaryDirectory() as first_temp, + tempfile.TemporaryDirectory() as second_temp, + ): + first_site = Path(first_temp) + second_site = Path(second_temp) + write_graph_site(first_site, label="First graph") + write_graph_site(second_site, label="Second graph") + + adapt_site(first_site, "/") + adapt_site(second_site, "/") + + first_index = (first_site / "index.html").read_text(encoding="utf-8") + second_index = (second_site / "index.html").read_text(encoding="utf-8") + self.assertNotEqual(graph_version(first_site), graph_version(second_site)) + self.assertNotEqual(first_index, second_index) + + def test_audit_rejects_stale_bundle_versions_after_data_change(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + write_graph_site(site_dir) + adapt_site(site_dir, "/") + old_key = graph_version(site_dir) + + graph_path = site_dir / "graph.json" + graph_path.write_text( + graph_path.read_text(encoding="utf-8").replace("Example", "Changed"), + encoding="utf-8", + ) + + violations = audit_site(site_dir, "/") + self.assertTrue( + any( + f"graph script URL /graph.js?v={old_key}" in violation + for violation in violations + ) + ) + self.assertTrue( + any( + f"graph data URL /graph.json?v={old_key}" in violation + for violation in violations + ) + ) + + def test_audit_rejects_unversioned_and_mismatched_urls(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + write_graph_site(site_dir) + adapt_site(site_dir, "/") + key = graph_version(site_dir) + + index_path = site_dir / "index.html" + index_path.write_text( + index_path.read_text(encoding="utf-8").replace( + f"/graph.js?v={key}", "/graph.js" + ), + encoding="utf-8", + ) + script_path = site_dir / "graph.js" + script_path.write_text( + script_path.read_text(encoding="utf-8").replace(key, "0" * 64), + encoding="utf-8", + ) + + violations = audit_site(site_dir, "/") + self.assertTrue( + any( + "graph script URL /graph.js (expected" in item + for item in violations + ) + ) + self.assertTrue( + any( + f"graph data URL /graph.json?v={'0' * 64} (expected" in item + for item in violations + ) + ) + + def test_audit_rejects_missing_graph_urls(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + site_dir = Path(temp_dir) + write_graph_site(site_dir) + (site_dir / "index.html").write_text( + "
No graph
", encoding="utf-8" + ) + (site_dir / "graph.js").write_text("const graph = true", encoding="utf-8") + + violations = audit_site(site_dir, "/") + self.assertIn("site: graph.js has no HTML script reference", violations) + self.assertIn("graph.js: missing graph.json fetch", violations) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_checkout_submodules.py b/tests/test_checkout_submodules.py new file mode 100644 index 0000000..343bcb9 --- /dev/null +++ b/tests/test_checkout_submodules.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "tools" / "checkout_submodules.py" + + +def command_environment() -> dict[str, str]: + environment = os.environ.copy() + environment["GIT_ALLOW_PROTOCOL"] = "file" + return environment + + +def run( + *arguments: str, + cwd: Path | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + arguments, + cwd=cwd, + check=check, + capture_output=True, + text=True, + env=command_environment(), + ) + + +def git( + repository: Path, + *arguments: str, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + return run("git", "-C", str(repository), *arguments, check=check) + + +def init_repository(path: Path) -> Path: + path.mkdir() + run("git", "init", "-b", "main", str(path)) + git(path, "config", "user.name", "Checkout Test") + git(path, "config", "user.email", "checkout@example.invalid") + return path + + +def commit_file( + repository: Path, + relative_path: str, + content: str, + message: str, +) -> str: + path = repository / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + git(repository, "add", relative_path) + git(repository, "commit", "-m", message) + return git(repository, "rev-parse", "HEAD").stdout.strip() + + +def add_submodule(parent: Path, source: Path, path: str) -> None: + git( + parent, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + source.resolve().as_uri(), + path, + ) + + +def is_shallow(repository: Path) -> bool: + result = git( + repository, + "rev-parse", + "--is-shallow-repository", + ) + return result.stdout.strip() == "true" + + +class NestedFixture: + def __init__(self, root: Path) -> None: + self.nested = init_repository(root / "nested-source") + self.nested_pin = commit_file( + self.nested, + "nested.txt", + "pinned nested content\n", + "add pinned nested content", + ) + commit_file( + self.nested, + "nested.txt", + "newer nested content\n", + "update nested content", + ) + + self.child = init_repository(root / "child-source") + add_submodule(self.child, self.nested, "vendor/nested") + git(self.child / "vendor/nested", "checkout", self.nested_pin) + git(self.child, "add", "vendor/nested") + git(self.child, "commit", "-m", "pin nested dependency") + self.child_pin = git( + self.child, + "rev-parse", + "HEAD", + ).stdout.strip() + self.child_newer = commit_file( + self.child, + "child.txt", + "newer child content\n", + "update child content", + ) + + self.parent = init_repository(root / "parent-source") + add_submodule(self.parent, self.child, "modules/child") + git(self.parent / "modules/child", "checkout", self.child_pin) + git(self.parent, "add", "modules/child") + git(self.parent, "commit", "-m", "pin child dependency") + + def clone(self, destination: Path, *, shallow: bool) -> Path: + arguments = ["git", "clone", "--recurse-submodules"] + if shallow: + arguments.extend(["--depth", "1", "--shallow-submodules"]) + arguments.extend([self.parent.resolve().as_uri(), str(destination)]) + run(*arguments) + return destination + + +class CheckoutSubmodulesTests(unittest.TestCase): + def run_helper( + self, + repository: Path, + *, + check: bool = True, + ) -> subprocess.CompletedProcess[str]: + return run( + sys.executable, + str(SCRIPT), + str(repository), + check=check, + ) + + def test_unshallows_top_level_and_nested_submodules(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fixture = NestedFixture(root) + clone = fixture.clone(root / "checkout", shallow=True) + child = clone / "modules/child" + nested = child / "vendor/nested" + + self.assertTrue(is_shallow(child)) + self.assertTrue(is_shallow(nested)) + + result = self.run_helper(clone) + + self.assertFalse(is_shallow(child)) + self.assertFalse(is_shallow(nested)) + self.assertEqual( + git(child, "rev-parse", "HEAD").stdout.strip(), + fixture.child_pin, + ) + self.assertEqual( + git(nested, "rev-parse", "HEAD").stdout.strip(), + fixture.nested_pin, + ) + self.assertIn("Verified 2 recursive submodule gitlinks", result.stdout) + + def test_restores_the_exact_parent_gitlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fixture = NestedFixture(root) + clone = fixture.clone(root / "checkout", shallow=False) + child = clone / "modules/child" + + git(child, "checkout", fixture.child_newer) + self.assertNotEqual( + git(child, "rev-parse", "HEAD").stdout.strip(), + fixture.child_pin, + ) + + self.run_helper(clone) + + self.assertEqual( + git(child, "rev-parse", "HEAD").stdout.strip(), + fixture.child_pin, + ) + + def test_fails_clearly_when_remote_lacks_the_gitlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + expected_source = init_repository(root / "expected-source") + expected = commit_file( + expected_source, + "expected.txt", + "expected content\n", + "add expected content", + ) + wrong_source = init_repository(root / "wrong-source") + commit_file( + wrong_source, + "wrong.txt", + "wrong content\n", + "add wrong content", + ) + + parent = init_repository(root / "parent-source") + add_submodule(parent, expected_source, "dependency") + git( + parent, + "config", + "-f", + ".gitmodules", + "submodule.dependency.url", + wrong_source.resolve().as_uri(), + ) + git(parent, "add", ".gitmodules", "dependency") + git(parent, "commit", "-m", "record unavailable dependency") + + clone = root / "checkout" + run("git", "clone", parent.resolve().as_uri(), str(clone)) + result = self.run_helper(clone, check=False) + + self.assertEqual(result.returncode, 1) + self.assertIn( + "Unable to check out every recorded recursive gitlink", + result.stderr, + ) + self.assertIn("dependency", result.stderr) + self.assertIn(expected, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_clean_local_stack.py b/tests/test_clean_local_stack.py new file mode 100644 index 0000000..d4d55db --- /dev/null +++ b/tests/test_clean_local_stack.py @@ -0,0 +1,668 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_tool(name: str): + path = ROOT / "tools" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +PREPARE = load_tool("prepare_local_stack") +SEED = load_tool("seed_local_pool") +CHECK = load_tool("check_local_stack") + + +def write_manifest(path: Path, *items: str | dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = [] + for item in items: + record = ( + { + "pid": item, + "schema_type": "xyzri:XYZProject", + } + if isinstance(item, str) + else item + ) + class_name = record["schema_type"].rsplit(":", 1)[-1] + lines.append( + json.dumps( + {"class_name": class_name, "record": record}, + sort_keys=True, + ) + ) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +class CleanLocalStackTests(unittest.TestCase): + def test_snapshot_restarts_with_one_page_size_and_checks_total(self) -> None: + def page(*pids: str, total: int, pages: int) -> dict: + return { + "total": total, + "pages": pages, + "items": [ + { + "pid": pid, + "schema_type": "xyzri:XYZProject", + } + for pid in pids + ], + } + + with tempfile.TemporaryDirectory() as temporary: + snapshot = Path(temporary) / "snapshot.jsonl" + responses = [ + (page("example:1", "example:2", total=4, pages=2), 100), + (page("example:3", total=4, pages=2), 50), + (page("example:1", "example:2", total=4, pages=2), 50), + (page("example:3", "example:4", total=4, pages=2), 50), + ] + with ( + mock.patch.object(PREPARE, "SNAPSHOT", snapshot), + mock.patch.object( + PREPARE, + "request_json", + return_value={"pid": "server"}, + ), + mock.patch.object( + PREPARE, + "fetch_page", + side_effect=responses, + ) as fetch_page, + ): + records, server = PREPARE.write_snapshot() + self.assertEqual(records, 4) + self.assertEqual(server, {"pid": "server"}) + self.assertEqual( + [call.args for call in fetch_page.call_args_list], + [(1, 100), (2, 100), (1, 50), (2, 50)], + ) + self.assertEqual(len(snapshot.read_text().splitlines()), 4) + + with ( + mock.patch.object(PREPARE, "SNAPSHOT", snapshot), + mock.patch.object( + PREPARE, + "request_json", + return_value={}, + ), + mock.patch.object( + PREPARE, + "fetch_page", + return_value=( + page("example:1", total=2, pages=1), + 100, + ), + ), + ): + with self.assertRaisesRegex(RuntimeError, "incomplete"): + PREPARE.write_snapshot() + self.assertEqual(len(snapshot.read_text().splitlines()), 4) + count, digest = PREPARE.snapshot_fingerprint(snapshot) + self.assertEqual(count, 4) + self.assertEqual(len(digest), 64) + + def test_prepare_isolates_collections_tokens_and_runtime_ui(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + stack = root / "build" / "local-stack" + source_ui = root / "source-ui" + source_ui.mkdir() + source_config = """service_base_url: + - url: http://127.0.0.1:8111/protected/ + type: write + - url: http://127.0.0.1:8111/public/ + type: read +token_info: Please contact Michael Hanke at m.hanke@fz-juelich.de for credentials. +use_service: true +""" + (source_ui / "config.yaml").write_text( + source_config, + encoding="utf-8", + ) + (source_ui / "config_default_xyzri.yaml").write_text( + "data_url: ''\n", + encoding="utf-8", + ) + schema = root / "schema.yaml" + schema.write_text("id: example:test\n", encoding="utf-8") + patches = ( + mock.patch.object(PREPARE, "STACK", stack), + mock.patch.object( + PREPARE, + "SERVICE_CONFIG", + stack / "dumpthings.yaml", + ), + mock.patch.object(PREPARE, "POOL_UI_SOURCE", source_ui), + mock.patch.object(PREPARE, "POOL_UI", stack / "ui"), + mock.patch.object(PREPARE, "SCHEMA", schema), + ) + with patches[0], patches[1], patches[2], patches[3], patches[4]: + stack.mkdir(parents=True) + PREPARE.write_service_config("editor-secret", "seed-secret") + for collection in PREPARE.LEGACY_COLLECTIONS: + legacy = stack / "store" / collection + legacy.mkdir() + (legacy / "record.json").write_text( + '{"pid": "example:legacy"}\n', + encoding="utf-8", + ) + removed = PREPARE.remove_legacy_collection_stores() + persisted = stack / "store" / "__dump_things__" + persisted.mkdir() + (persisted / "stale-config").write_text( + "old two-collection config\n", + encoding="utf-8", + ) + PREPARE.reset_persisted_service_config() + PREPARE.prepare_pool_ui() + + service = (stack / "dumpthings.yaml").read_text(encoding="utf-8") + for collection in PREPARE.COLLECTIONS: + self.assertIn(f" {collection}:\n", service) + self.assertTrue((stack / "store" / collection / "curated").is_dir()) + self.assertTrue((stack / "store" / collection / "incoming").is_dir()) + self.assertEqual(service.count("default_token: local_reader"), 3) + self.assertEqual(service.count("default_token: local_con_reader"), 1) + self.assertNotIn("local_denied", service) + self.assertEqual( + {path.name for path in removed}, + set(PREPARE.LEGACY_COLLECTIONS), + ) + for collection in PREPARE.LEGACY_COLLECTIONS: + self.assertFalse((stack / "store" / collection).exists()) + self.assertFalse((stack / "store" / "__dump_things__").exists()) + reader = service.split(" local_reader:", 1)[1].split( + " local_con_reader:", 1 + )[0] + self.assertIn("upstream-public:", reader) + self.assertIn("upstream-protected:", reader) + self.assertIn("con-public:", reader) + self.assertNotIn("con-protected:", reader) + self.assertNotIn("WRITE", reader) + con_reader = service.split(" local_con_reader:", 1)[1].split( + " local_editor:", 1 + )[0] + self.assertIn("user_id: local-con-reader", con_reader) + self.assertIn("con-protected:", con_reader) + self.assertIn("mode: READ_CURATED", con_reader) + self.assertNotIn("upstream-", con_reader) + self.assertNotIn("WRITE", con_reader) + self.assertEqual(con_reader.count("mode:"), 1) + editor = service.split(" local_editor:", 1)[1].split(" local_seeder:", 1)[ + 0 + ] + self.assertIn("con-protected:", editor) + self.assertIn("mode: WRITE_COLLECTION", editor) + self.assertIn("incoming_label: local-editor", editor) + self.assertNotIn("upstream-", editor) + self.assertNotIn("con-public:", editor) + self.assertNotIn("READ_", editor) + self.assertEqual(editor.count("mode:"), 1) + seeder = service.split(" local_seeder:", 1)[1] + for collection in PREPARE.COLLECTIONS: + self.assertIn(f" {collection}:\n", seeder) + runtime = (stack / "ui" / "config.yaml").read_text(encoding="utf-8") + con_url = "http://127.0.0.1:8111/con-protected/" + self.assertEqual(runtime.count(con_url), 2) + self.assertNotIn("127.0.0.1:8111/public/", runtime) + self.assertNotIn("127.0.0.1:8111/protected/", runtime) + self.assertIn("build/local-stack/editor-token", runtime) + self.assertNotIn("Please contact Michael Hanke", runtime) + self.assertEqual( + (source_ui / "config.yaml").read_text(encoding="utf-8"), + source_config, + ) + + def test_seed_manifests_target_only_their_collection_pairs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + upstream = root / "upstream.jsonl" + con = root / "con.jsonl" + write_manifest(upstream, "example:upstream-1", "example:upstream-2") + write_manifest(con, "example:con-1") + with ( + mock.patch.object( + SEED, + "put_record", + return_value="created", + ) as put_record, + mock.patch.object( + SEED, + "prune_collection", + return_value=0, + ) as prune, + ): + SEED.seed_manifest( + upstream, + SEED.UPSTREAM_COLLECTIONS, + "seed-token", + "upstream", + ) + SEED.seed_manifest( + con, + SEED.CON_COLLECTIONS, + "seed-token", + "CON", + ) + targets: dict[str, set[str]] = {} + for call in put_record.call_args_list: + collection, _, record, _ = call.args + targets.setdefault(record["pid"], set()).add(collection) + self.assertEqual( + targets["example:upstream-1"], + set(SEED.UPSTREAM_COLLECTIONS), + ) + self.assertEqual( + targets["example:upstream-2"], + set(SEED.UPSTREAM_COLLECTIONS), + ) + self.assertEqual( + targets["example:con-1"], + set(SEED.CON_COLLECTIONS), + ) + self.assertEqual( + [call.args[0] for call in prune.call_args_list], + [ + "upstream-public", + "upstream-protected", + "con-public", + "con-protected", + ], + ) + + def test_manifest_rejects_duplicate_pids(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + manifest = Path(temporary) / "records.jsonl" + write_manifest(manifest, "example:same", "example:same") + with self.assertRaisesRegex(RuntimeError, "duplicate record pid"): + SEED.load_manifest(manifest) + + def test_seed_idempotence_accepts_service_class_normalization(self) -> None: + record = { + "pid": "example:project", + "schema_type": "xyzri:XYZProject", + } + with mock.patch.object( + SEED, + "call", + return_value=(200, {"pid": "example:project"}), + ) as call: + result = SEED.put_record( + "con-public", + "XYZProject", + record, + "seed-token", + ) + self.assertEqual(result, "unchanged") + self.assertEqual(call.call_count, 1) + + def test_check_compares_all_four_curated_payloads(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + upstream = root / "upstream.jsonl" + con = root / "con.jsonl" + write_manifest( + upstream, + { + "pid": "example:shared", + "schema_type": "xyzri:XYZProject", + "title": "Upstream", + }, + ) + write_manifest( + con, + { + "pid": "example:shared", + "schema_type": "xyzri:XYZProject", + "title": "CON", + }, + ) + upstream_records = CHECK.manifest_records(upstream) + con_records = CHECK.manifest_records(con) + + def records(collection: str, _token: str) -> dict[str, dict]: + if collection.startswith("upstream-"): + return upstream_records + return con_records + + with ( + mock.patch.object(CHECK, "UPSTREAM_SNAPSHOT", upstream), + mock.patch.object(CHECK, "CON_RECORDS", con), + mock.patch.object(CHECK, "curated_records", side_effect=records), + ): + counts = CHECK.check_seed_separation("seed-token") + self.assertEqual(counts["upstream-public"], 1) + self.assertEqual(counts["con-protected"], 1) + + def contaminated( + collection: str, + token: str, + ) -> dict[str, dict]: + if collection == "con-public": + return upstream_records + return records(collection, token) + + with ( + mock.patch.object(CHECK, "UPSTREAM_SNAPSHOT", upstream), + mock.patch.object(CHECK, "CON_RECORDS", con), + mock.patch.object( + CHECK, + "curated_records", + side_effect=contaminated, + ), + ): + with self.assertRaisesRegex( + RuntimeError, + "con-public.*changed", + ): + CHECK.check_seed_separation("seed-token") + + def test_write_probe_is_confined_to_con_protected_incoming(self) -> None: + calls: list[tuple[str, str, str]] = [] + written: dict | None = None + + def request( + method: str, + url: str, + token: str | None, + body=None, + *, + missing_ok: bool = False, + ): + nonlocal written + del missing_ok + calls.append((method, url, token or "")) + if method == "DELETE": + return None + if method == "POST": + written = body + return body + if written is not None and "/con-protected/incoming/local-editor/" in url: + return written + return None + + with ( + mock.patch.object(CHECK, "request_json", side_effect=request), + mock.patch.object(CHECK, "expect_rejected") as rejected, + mock.patch.object( + CHECK, + "manifest_envelopes", + return_value={"xyzrins:.": {}}, + ), + mock.patch.object( + CHECK, + "incoming_probe_pids", + return_value={f"{CHECK.PROBE_PID_PREFIX}stale"}, + ), + mock.patch.object( + CHECK.uuid, + "uuid4", + return_value=mock.Mock(hex="unit-probe"), + ), + ): + CHECK.prove_write_isolation("editor-token", "seed-token") + post = next(call for call in calls if call[0] == "POST") + self.assertEqual(post[0], "POST") + self.assertIn("/con-protected/record/XYZProject", post[1]) + self.assertEqual(post[2], "editor-token") + self.assertEqual(rejected.call_count, 7) + probe = { + "pid": ("xyzrins:projects/_clean-migration-write-probe-unit-probe"), + "schema_type": "xyzri:XYZProject", + } + for collection in ( + "upstream-public", + "upstream-protected", + "con-public", + "con-protected", + ): + url = f"http://127.0.0.1:8111/{collection}/record/XYZProject" + rejected.assert_any_call("POST", url, None, probe) + for collection in ( + "upstream-public", + "upstream-protected", + "con-public", + ): + rejected.assert_any_call( + "POST", + f"http://127.0.0.1:8111/{collection}/record/XYZProject", + "editor-token", + probe, + ) + self.assertGreaterEqual( + sum(call[0] == "DELETE" for call in calls), + 24, + ) + self.assertTrue( + any(call[0] == "DELETE" and "stale" in call[1] for call in calls) + ) + + def test_write_probe_namespace_cannot_collide_with_canonical_data(self) -> None: + collision = f"{CHECK.PROBE_PID_PREFIX}canonical" + with mock.patch.object( + CHECK, + "manifest_envelopes", + return_value={collision: {}}, + ): + with self.assertRaisesRegex(RuntimeError, "reserved acceptance PID"): + CHECK.prove_write_isolation("editor-token", "seed-token") + + def test_stale_write_probes_are_discovered_without_human_edits(self) -> None: + stale = f"{CHECK.PROBE_PID_PREFIX}interrupted" + pages = ( + { + "items": [ + {"pid": "xyzrins:projects/human-pending-edit"}, + {"pid": CHECK.LEGACY_PROBE_PID}, + ], + "pages": 2, + }, + {"items": [{"pid": stale}], "pages": 2}, + ) + with mock.patch.object(CHECK, "request_json", side_effect=pages) as request: + self.assertEqual( + CHECK.incoming_probe_pids("con-protected", "seed-token"), + {CHECK.LEGACY_PROBE_PID, stale}, + ) + self.assertEqual(request.call_count, 2) + + def test_ui_links_and_supervisor_default_to_con(self) -> None: + config = """use_service: true +use_token: true +service_base_url: + - url: http://127.0.0.1:8111/con-protected/ + type: write + - url: http://127.0.0.1:8111/con-protected/ + type: read +gitannex_p2phttp_url: http://127.0.0.1:8122/git-annex +""" + external = "xyzrins:\ndlschemas_owl.ttl\ndata_url: ''\n" + expected_pids = frozenset( + { + "xyzrins:.", + CHECK.CON_PERSON_PID, + "xyzrins:projects/datalad", + "xyzrins:persons/new-member", + } + ) + with tempfile.TemporaryDirectory() as temporary: + site = Path(temporary) + for index, pid in enumerate(sorted(expected_pids)): + page = site / str(index) + page.mkdir() + query_pid = pid.replace(":", "%3A").replace("/", "%2F") + (page / "index.html").write_text( + 'edit', + encoding="utf-8", + ) + with ( + mock.patch.object( + CHECK, + "read_text", + side_effect=(config, external), + ), + mock.patch.object(CHECK, "CON_SITE", site), + ): + CHECK.check_editor_ui() + self.assertEqual( + CHECK.check_static_edit_links(expected_pids), + len(expected_pids), + ) + + def test_edit_links_reject_credentials_and_unknown_records(self) -> None: + def write_link(site: Path, query: str) -> None: + (site / "index.html").write_text( + f'edit', + encoding="utf-8", + ) + + with tempfile.TemporaryDirectory() as temporary: + site = Path(temporary) + credential_query = ( + "sh%3ANodeShape=dlthings%3AThing&" + "pid=xyzrins%3Apersons%2Fyaroslav-halchenko&" + "edit=true&token=secret" + ) + write_link(site, credential_query) + with mock.patch.object(CHECK, "CON_SITE", site): + with self.assertRaisesRegex( + RuntimeError, + "credential-free", + ): + CHECK.check_static_edit_links(frozenset({CHECK.CON_PERSON_PID})) + + unknown_query = ( + "sh%3ANodeShape=dlthings%3AThing&" + "pid=xyzrins%3Apersons%2Funknown&edit=true" + ) + write_link(site, unknown_query) + with mock.patch.object(CHECK, "CON_SITE", site): + with self.assertRaisesRegex( + RuntimeError, + "rendered record set", + ): + CHECK.check_static_edit_links(frozenset({CHECK.CON_PERSON_PID})) + + def test_edit_pid_closure_comes_from_records_and_render_policy(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + records = root / "records.jsonl" + projection = root / "projection.yaml" + write_manifest( + records, + { + "pid": "xyzrins:.", + "schema_type": "xyzri:XYZProject", + }, + { + "pid": "xyzrins:projects/datalad", + "schema_type": "xyzri:XYZProject", + }, + { + "pid": CHECK.CON_PERSON_PID, + "schema_type": "xyzri:XYZPerson", + }, + { + "pid": "xyzrins:persons/new-member", + "schema_type": "xyzri:XYZPerson", + }, + { + "pid": "marcrel:aut", + "schema_type": "xyzri:XYZAgentRole", + }, + ) + projection.write_text( + """render: + pages: + xyzri:XYZProject: project.md.j2 + xyzri:XYZPerson: person.md.j2 + homepage: + pid: xyzrins:. +""", + encoding="utf-8", + ) + self.assertEqual( + CHECK.expected_edit_pids(records, projection), + { + "xyzrins:.", + "xyzrins:projects/datalad", + CHECK.CON_PERSON_PID, + "xyzrins:persons/new-member", + }, + ) + + without_datalad = root / "without-datalad.jsonl" + write_manifest( + without_datalad, + { + "pid": "xyzrins:.", + "schema_type": "xyzri:XYZProject", + }, + { + "pid": CHECK.CON_PERSON_PID, + "schema_type": "xyzri:XYZPerson", + }, + ) + with self.assertRaisesRegex(RuntimeError, "Representative"): + CHECK.expected_edit_pids(without_datalad, projection) + + def test_anonymous_read_targets_curated_yaroslav_record(self) -> None: + record = {"pid": CHECK.CON_PERSON_PID} + with mock.patch.object( + CHECK, + "request_json", + return_value=record, + ) as request: + CHECK.check_anonymous_con_read() + request.assert_called_once_with( + "GET", + ( + "http://127.0.0.1:8111/con-protected/record?" + "pid=xyzrins%3Apersons%2Fyaroslav-halchenko&format=json" + ), + None, + ) + + with mock.patch.object(CHECK, "request_json", return_value=None): + with self.assertRaisesRegex(RuntimeError, "Anonymous"): + CHECK.check_anonymous_con_read() + + supervisor = (ROOT / "tools" / "serve_local_stack.sh").read_text( + encoding="utf-8" + ) + self.assertIn('--directory "$stack_dir/ui"', supervisor) + self.assertIn('--directory "$root_dir/build/con-site"', supervisor) + self.assertNotIn("build/upstream-local", supervisor) + self.assertIn("trap cleanup EXIT", supervisor) + self.assertIn("trap 'exit 130' INT TERM", supervisor) + + def test_check_rejects_legacy_collection_stores(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + stack = Path(temporary) + with mock.patch.object(CHECK, "STACK", stack): + CHECK.check_no_legacy_collection_stores() + legacy = stack / "store" / "public" + legacy.mkdir(parents=True) + with self.assertRaisesRegex(RuntimeError, "Obsolete"): + CHECK.check_no_legacy_collection_stores() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_clean_projection_contract.py b/tests/test_clean_projection_contract.py new file mode 100644 index 0000000..0945247 --- /dev/null +++ b/tests/test_clean_projection_contract.py @@ -0,0 +1,1104 @@ +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import call, patch + + +ROOT = Path(__file__).resolve().parents[1] +TOOLS = ROOT / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +import build_con_site as BUILD # noqa: E402 +import con_projection as PROJECTION # noqa: E402 + + +class CleanProjectionContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + if not PROJECTION.PROFILE_PATH.is_file(): + raise unittest.SkipTest("clean-migration site gitlink is not pinned") + + def projection_contract(self) -> PROJECTION.ProjectionContract: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + graph = specification["graph"] + producer = PROJECTION.site_manifest_path( + graph["producer"], "projection.graph.producer" + ) + graph.setdefault( + "node_classes", + sorted(PROJECTION.producer_mapping(producer, "wanted_node_types")), + ) + graph.setdefault( + "relationship_fields", + sorted(PROJECTION.producer_mapping(producer, "wanted_edge_types")), + ) + return PROJECTION.load_projection_contract(profile, specification) + + def source_closure(self) -> list[PROJECTION.SourceRecord]: + return PROJECTION.source_closure(self.projection_contract()) + + def test_terminal_snapshot_owns_projection_and_assembly_outputs(self) -> None: + self.assertTrue( + PROJECTION.generated_snapshot_path("profiles/con/projection/records.jsonl") + ) + self.assertTrue( + PROJECTION.generated_snapshot_path("profiles/con/assembly/SHA256SUMS") + ) + self.assertFalse( + PROJECTION.generated_snapshot_path("profiles/con/presentation.yaml") + ) + + def test_successor_history_preserves_checkpoint_and_accepts_focused_commits( + self, + ) -> None: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + PROJECTION.verify_successor_history(profile) + base = profile["components"]["www_from_model"]["commit"] + subjects = subprocess.run( + [ + "git", + "-C", + str(PROJECTION.SITE), + "log", + "--reverse", + "--format=%s", + f"{base}..HEAD", + ], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + self.assertGreaterEqual(len(subjects), 2) + self.assertEqual( + subjects[:2], + list(PROJECTION.FOUNDATION_SUBJECTS), + ) + self.assertIn( + PROJECTION.ACCEPTED_CLEAN_MIGRATION_TIP, + PROJECTION.checkpoint_refs().values(), + ) + self.assertIn( + PROJECTION.ACCEPTED_CLEAN_MIGRATION_PARENT_TIP, + PROJECTION.checkpoint_refs(ROOT).values(), + ) + + def test_final_site_state_requires_terminal_history_and_clean_checkout( + self, + ) -> None: + profile = {"profile": "fixture"} + repository = Path("fixture-site") + with ( + patch.object(PROJECTION, "verify_successor_history") as history, + patch.object(PROJECTION, "require_clean_checkout") as clean, + patch.object(PROJECTION, "require_no_ignored_files") as no_ignored, + ): + PROJECTION.verify_final_site_state(profile, repository) + history.assert_called_once_with( + profile, + repository, + require_terminal=True, + ) + self.assertEqual( + clean.call_args_list, + [ + call(repository, "full-migration website"), + call(PROJECTION.ROOT, "full-migration coordinator"), + ], + ) + self.assertEqual( + no_ignored.call_args_list, + [ + call(repository, "full-migration website"), + call( + PROJECTION.UPSTREAM, + "www-from-model hydration transport", + ("assets", "static", "themes/congo"), + ), + ], + ) + + def test_terminal_history_has_distinct_preparation_and_final_modes(self) -> None: + PROJECTION.verify_terminal_history([], 4, require_terminal=False) + PROJECTION.verify_terminal_history([3], 4, require_terminal=True) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "exactly one terminal generated projection commit", + ): + PROJECTION.verify_terminal_history([], 4, require_terminal=True) + for indexes in ([2], [2, 3]): + with ( + self.subTest(indexes=indexes), + self.assertRaisesRegex( + PROJECTION.ProjectionError, + "unique and terminal", + ), + ): + PROJECTION.verify_terminal_history( + indexes, + 4, + require_terminal=False, + ) + + def test_snapshot_check_enters_final_site_mode(self) -> None: + profile = {"profile": "fixture"} + with ( + patch.object( + PROJECTION.sys, + "argv", + ["con_projection.py", "check-snapshot"], + ), + patch.object(PROJECTION, "load_yaml", return_value=profile), + patch.object( + PROJECTION, + "verify_final_site_state", + side_effect=PROJECTION.ProjectionError("strict marker"), + ) as final_state, + patch("builtins.print"), + ): + self.assertEqual(PROJECTION.main(), 1) + final_state.assert_called_once_with(profile) + + def test_static_build_enters_final_site_mode(self) -> None: + profile = {"profile": "fixture"} + destination = Path("fixture-site") + with ( + patch.object(BUILD, "safe_destination", return_value=destination), + patch.object(BUILD, "load_yaml", return_value=profile), + patch.object( + BUILD, + "verify_final_site_state", + side_effect=PROJECTION.ProjectionError("strict marker"), + ) as final_state, + ): + with self.assertRaisesRegex(BUILD.BuildError, "strict marker"): + BUILD.build_site(destination, "http://127.0.0.1:8767/") + final_state.assert_called_once_with(profile) + + def test_dirty_site_policy_rejects_upstream_owned_paths(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + (repository / "layouts").mkdir() + (repository / "layouts" / "upstream.html").write_text( + "upstream\n", encoding="utf-8" + ) + subprocess.run( + ["git", "init", "--initial-branch=main", str(repository)], + check=True, + capture_output=True, + ) + subprocess.run( + ["git", "-C", str(repository), "add", "."], + check=True, + capture_output=True, + ) + subprocess.run( + [ + "git", + "-C", + str(repository), + "-c", + "user.name=Projection Test", + "-c", + "user.email=projection@example.invalid", + "commit", + "-m", + "test: create fixture", + ], + check=True, + capture_output=True, + ) + for relative in ( + ".gitmodules", + "config/con/hugo.yaml", + "profiles/con/metadata/person.yaml", + ): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("fixture\n", encoding="utf-8") + PROJECTION.verify_site_worktree_isolation(repository) + + (repository / "layouts" / "upstream.html").write_text( + "downstream edit\n", encoding="utf-8" + ) + (repository / "content" / "leaked.md").parent.mkdir() + (repository / "content" / "leaked.md").write_text( + "leak\n", encoding="utf-8" + ) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "dirty or untracked upstream-owned paths", + ): + PROJECTION.verify_site_worktree_isolation(repository) + + def test_final_input_policy_rejects_ignored_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + subprocess.run( + ["git", "init", "--initial-branch=main", str(repository)], + check=True, + capture_output=True, + ) + (repository / ".gitignore").write_text( + "resources/\n", + encoding="utf-8", + ) + subprocess.run( + ["git", "-C", str(repository), "add", ".gitignore"], + check=True, + capture_output=True, + ) + hidden = repository / "profiles/con/metadata/resources/hidden.yaml" + hidden.parent.mkdir(parents=True) + hidden.write_text("schema_type: xyzri:XYZPerson\n", encoding="utf-8") + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "ignored files", + ): + PROJECTION.require_no_ignored_files(repository, "fixture") + PROJECTION.require_no_ignored_files( + repository, + "scoped fixture", + ("assets",), + ) + scoped = repository / "assets/resources/hidden.yaml" + scoped.parent.mkdir(parents=True) + scoped.write_text("hidden: true\n", encoding="utf-8") + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "ignored files", + ): + PROJECTION.require_no_ignored_files( + repository, + "scoped fixture", + ("assets",), + ) + + def test_successor_history_rejects_merge_parents(self) -> None: + base = "0" * 40 + first = "1" * 40 + merge = "2" * 40 + other = "3" * 40 + with patch.object( + PROJECTION, + "run", + side_effect=[f"{base}\n", f"{first} {other}\n"], + ): + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "linear commit stack", + ): + PROJECTION.verify_linear_successor_history( + Path("fixture-site"), + base, + [first, merge], + ) + + def test_native_metadata_and_committed_projection_close_exactly( + self, + ) -> None: + contract = self.projection_contract() + records = self.source_closure() + expectations = PROJECTION.validate_record_contract(records, contract) + observed = PROJECTION.native_value_fingerprint( + [item.record for item in records] + ) + self.assertTrue( + PROJECTION.REQUIRED_NATIVE_TYPES + <= {schema_type for schema_type, _ in observed}, + ) + self.assertIn( + ( + "dlthings:DOI", + PROJECTION.normalized_payload( + { + "notation": "10.21105/joss.03262", + "schema_type": "dlthings:DOI", + } + ), + ), + observed, + ) + self.assertIn( + ( + "dlthings:ISSN", + PROJECTION.normalized_payload( + { + "notation": "2475-9066", + "schema_type": "dlthings:ISSN", + } + ), + ), + observed, + ) + + qualified = { + "schema_type": "dlthings:Association", + "object": "xyzrins:persons/example", + "roles": ["marcrel:led"], + "statement": "qualified relationship", + } + changed = deepcopy(qualified) + changed["statement"] = "changed qualifier" + self.assertNotEqual( + PROJECTION.native_value_fingerprint(qualified), + PROJECTION.native_value_fingerprint(changed), + ) + + # The accepted slice remains a representative semantic smoke fixture, + # not the production validator's complete PID/edge/page inventory. + self.assertTrue( + { + "xyzrins:.", + "ror:04tfhh831", + "xyzrins:persons/yaroslav-halchenko", + "xyzrins:projects/datalad", + "xyzrins:publications/datalad-joss-2021", + "xyzrins:instruments/datalad", + } + <= expectations.canonical_pids + ) + self.assertTrue( + { + "marcrel:led", + "marcrel:aut", + "obo:IAO_0000010", + "bibo:AcademicArticle", + } + <= expectations.reference_pids + ) + self.assertIn( + ( + "xyzrins:publications/datalad-joss-2021", + "xyzrins:projects/datalad", + ), + expectations.graph_edges, + ) + self.assertIn( + "persons/yaroslav-halchenko", + expectations.entity_routes, + ) + + by_pid = {item.record["pid"]: item.record for item in records} + self.assertEqual( + by_pid["xyzrins:projects/datalad"]["part_of"], + ["xyzrins:."], + ) + publication = by_pid["xyzrins:publications/datalad-joss-2021"] + self.assertFalse( + any( + str(identifier.get("notation", "")).startswith("https://doi.org/") + for identifier in publication.get("identifiers", []) + if isinstance(identifier, dict) + ) + ) + + PROJECTION.verify_manifest(PROJECTION.COMMITTED) + snapshot = [ + json.loads(line) + for line in (PROJECTION.COMMITTED / "records.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ] + report = PROJECTION.validate_projection( + snapshot, PROJECTION.COMMITTED, expectations + ) + self.assertEqual(report["graph_nodes"], len(expectations.graph_node_pids)) + self.assertEqual(report["graph_edges"], len(expectations.graph_edges)) + self.assertEqual(report["pages"], len(expectations.markdown_pages)) + + def test_source_inventory_expands_pages_and_graph_without_code_edits( + self, + ) -> None: + contract = self.projection_contract() + records = deepcopy(self.source_closure()) + homepage = next(item for item in records if item.record["pid"] == "xyzrins:.") + homepage.record["associated_with"].append( + { + "object": "xyzrins:persons/example-person", + "schema_type": "dlthings:Association", + } + ) + records.append( + PROJECTION.SourceRecord( + class_name="XYZPerson", + record={ + "pid": "xyzrins:persons/example-person", + "schema_type": "xyzri:XYZPerson", + "given_name": "Example", + "family_name": "Person", + "display_label": "Example Person", + }, + path=contract.canonical_root / "XYZPerson" / "example-person.yaml", + category="canonical", + ) + ) + expectations = PROJECTION.validate_record_contract(records, contract) + self.assertIn( + "persons/example-person/_index.md", + expectations.markdown_pages, + ) + self.assertIn( + ("xyzrins:.", "xyzrins:persons/example-person"), + expectations.graph_edges, + ) + + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + for page in expectations.markdown_pages: + path = output / "content" / page + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("---\n---\n", encoding="utf-8") + graph_path = output / "static" / "graph.json" + graph_path.parent.mkdir(parents=True) + graph_json = json.dumps( + { + "nodes": [ + {"id": pid} for pid in sorted(expectations.graph_node_pids) + ], + "edges": [ + {"source": source, "target": target} + for source, target in sorted(expectations.graph_edges) + ], + } + ) + graph_path.write_text(graph_json, encoding="utf-8") + (output / "graph.json").write_text(graph_json, encoding="utf-8") + report = PROJECTION.validate_projection( + [item.record for item in records], output, expectations + ) + altered = [deepcopy(item.record) for item in records] + altered[0]["projection_test_qualifier"] = "changed" + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "record payload differs from source inventory", + ): + PROJECTION.validate_projection(altered, output, expectations) + for route in expectations.entity_routes: + route_index = output / route / "index.html" + route_index.parent.mkdir(parents=True, exist_ok=True) + route_index.write_text("\n", encoding="utf-8") + BUILD.graph_contract(output, expectations) + self.assertEqual( + BUILD.entity_routes(output, expectations.entity_routes), + expectations.entity_routes, + ) + self.assertEqual(report["canonical_records"], len(expectations.canonical_pids)) + + def test_invalid_discriminators_bridges_and_targets_are_rejected( + self, + ) -> None: + contract = self.projection_contract() + original = self.source_closure() + + def replace_association( + value: object, + replacement: str, + ) -> bool: + if isinstance(value, dict): + if value.get("schema_type") == "dlthings:Association": + value["schema_type"] = replacement + return True + return any( + replace_association(child, replacement) for child in value.values() + ) + if isinstance(value, list): + return any(replace_association(child, replacement) for child in value) + return False + + for replacement, message in ( + ( + "https://concepts.datalad.org/s/things/v2/Association", + "full-URI", + ), + ("dlthings:NotARealAssociation", "unknown CURIE"), + ): + records = deepcopy(original) + self.assertTrue( + replace_association(records[0].record, replacement) + or any( + replace_association(item.record, replacement) + for item in records[1:] + ) + ) + with self.assertRaisesRegex(PROJECTION.ProjectionError, message): + PROJECTION.validate_record_contract(records, contract) + + dangling = deepcopy(original) + project = next( + item + for item in dangling + if item.record["pid"] == "xyzrins:projects/datalad" + ) + project.record["associated_with"][0]["object"] = "xyzrins:missing" + with self.assertRaisesRegex(PROJECTION.ProjectionError, "dangling"): + PROJECTION.validate_record_contract(dangling, contract) + + native_reference = deepcopy(original) + project = next( + item + for item in native_reference + if item.record["pid"] == "xyzrins:projects/datalad" + ) + project.record["part_of"] = ["marcrel:led"] + with self.assertRaisesRegex(PROJECTION.ProjectionError, "native graph target"): + PROJECTION.validate_record_contract(native_reference, contract) + + bridge = deepcopy(original) + project = next( + item for item in bridge if item.record["pid"] == "xyzrins:projects/datalad" + ) + project.record.setdefault("attributes", []).append( + { + "predicate": "dcterms:relation", + "value": "xyzrins:missing", + "schema_type": "dlthings:AttributeSpecification", + } + ) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "cannot encode relationship", + ): + PROJECTION.validate_record_contract(bridge, contract) + + unused_reference = deepcopy(original) + unused_reference.append( + PROJECTION.SourceRecord( + class_name="XYZAgentRole", + record={ + "pid": "marcrel:unused", + "schema_type": "xyzri:XYZAgentRole", + "display_label": "Unused", + }, + path=contract.reference_root / "XYZAgentRole" / "unused.yaml", + category="reference", + ) + ) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, "outside the canonical native-link" + ): + PROJECTION.validate_record_contract(unused_reference, contract) + + def test_graph_traversal_must_be_declared_and_match_producer(self) -> None: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + producer = PROJECTION.site_manifest_path( + specification["graph"]["producer"], + "projection.graph.producer", + ) + specification["graph"].setdefault( + "node_classes", + sorted(PROJECTION.producer_mapping(producer, "wanted_node_types")), + ) + specification["graph"].setdefault( + "relationship_fields", + sorted(PROJECTION.producer_mapping(producer, "wanted_edge_types")), + ) + for field in ("node_classes", "relationship_fields"): + missing = deepcopy(specification) + missing["graph"].pop(field, None) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + f"projection.graph.{field}", + ): + PROJECTION.load_projection_contract(profile, missing) + + mismatched = deepcopy(specification) + mismatched["graph"]["node_classes"] = ["xyzri:XYZPerson"] + mismatched["graph"]["relationship_fields"] = ["part_of"] + with self.assertRaisesRegex(PROJECTION.ProjectionError, "pinned producer"): + PROJECTION.load_projection_contract(profile, mismatched) + + def test_profile_paths_homepage_and_page_policy_are_executable( + self, + ) -> None: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + producer = PROJECTION.site_manifest_path( + specification["graph"]["producer"], + "projection.graph.producer", + ) + specification["graph"].setdefault( + "node_classes", + sorted(PROJECTION.producer_mapping(producer, "wanted_node_types")), + ) + specification["graph"].setdefault( + "relationship_fields", + sorted(PROJECTION.producer_mapping(producer, "wanted_edge_types")), + ) + + wrong_path = deepcopy(profile) + wrong_path["paths"]["canonical_records"] = wrong_path["paths"][ + "reference_records" + ] + with self.assertRaisesRegex( + PROJECTION.ProjectionError, "canonical_records paths disagree" + ): + PROJECTION.load_projection_contract(wrong_path, specification) + + wrong_homepage = deepcopy(profile) + wrong_homepage["identity"]["homepage_pid"] = "xyzrins:projects/not-the-homepage" + with self.assertRaisesRegex( + PROJECTION.ProjectionError, "disagree on homepage PID" + ): + PROJECTION.load_projection_contract(wrong_homepage, specification) + + no_person_pages = deepcopy(specification) + no_person_pages["render"]["pages"].pop("xyzri:XYZPerson") + no_person_pages["render"]["unrendered_classes"].append("xyzri:XYZPerson") + contract = PROJECTION.load_projection_contract(profile, no_person_pages) + expectations = PROJECTION.validate_record_contract( + PROJECTION.source_closure(contract), contract + ) + self.assertNotIn("persons/yaroslav-halchenko", expectations.entity_routes) + self.assertIn( + "xyzrins:persons/yaroslav-halchenko", + expectations.graph_node_pids, + ) + + def test_runtime_declarations_are_enforced_exactly(self) -> None: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + cases = ( + ( + "profile.schema.path", + lambda candidate_profile, _: candidate_profile["schema"].__setitem__( + "path", "src/demo-research-information/resolved.yaml" + ), + ), + ( + "profile.paths.qri_snapshot", + lambda candidate_profile, _: candidate_profile["paths"].__setitem__( + "qri_snapshot", "profiles/con/projection/wrong.jsonl" + ), + ), + ( + "projection.snapshot.records", + lambda _, candidate: candidate["snapshot"].__setitem__( + "records", "profiles/con/projection/wrong.jsonl" + ), + ), + ( + "projection.snapshot.format", + lambda _, candidate: candidate["snapshot"].__setitem__( + "format", "yaml" + ), + ), + ( + "projection.snapshot.sort_key", + lambda _, candidate: candidate["snapshot"].__setitem__( + "sort_key", ["pid"] + ), + ), + ( + "projection.render.engine", + lambda _, candidate: candidate["render"].__setitem__( + "engine", "custom" + ), + ), + ( + "projection.render.content_root", + lambda _, candidate: candidate["render"].__setitem__( + "content_root", "profiles/con/projection/wrong-content" + ), + ), + ( + "projection.graph.output", + lambda _, candidate: candidate["graph"].__setitem__( + "output", "profiles/con/projection/wrong-graph.json" + ), + ), + ( + "projection.digest.algorithm", + lambda _, candidate: candidate["digest"].__setitem__( + "algorithm", "sha512" + ), + ), + ( + "projection.digest.output", + lambda _, candidate: candidate["digest"].__setitem__( + "output", "profiles/con/projection/wrong-digest" + ), + ), + ) + for label, mutate in cases: + with self.subTest(label=label): + candidate_profile = deepcopy(profile) + candidate_specification = deepcopy(specification) + mutate(candidate_profile, candidate_specification) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + label.replace(".", r"\."), + ): + PROJECTION.load_projection_contract( + candidate_profile, candidate_specification + ) + + def test_all_pinned_upstream_page_classes_have_executable_pipelines( + self, + ) -> None: + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + specification["render"]["pages"].update( + { + "xyzri:XYZDataset": "page_templates/dataset.md.j2", + "xyzri:XYZObjective": "page_templates/objective.md.j2", + "xyzri:XYZTopic": "page_templates/topic.md.j2", + } + ) + contract = PROJECTION.load_projection_contract(profile, specification) + pages, homepage = PROJECTION.upstream_qri_pipelines(contract) + self.assertEqual( + set(pages), + { + "xyzri:XYZDataset", + "xyzri:XYZInstrument", + "xyzri:XYZObjective", + "xyzri:XYZPerson", + "xyzri:XYZProject", + "xyzri:XYZPublication", + "xyzri:XYZTopic", + }, + ) + self.assertEqual( + pages["xyzri:XYZObjective"], + [ + ["qri", "list", "--class", "xyzri:XYZObjective"], + [ + "qri", + "inline-records", + "-p", + "part_of", + "-c", + "con-public", + ], + [ + "qri", + "inline-records", + "-p", + "depends_on", + "-c", + "con-public", + ], + ], + ) + self.assertEqual( + pages["xyzri:XYZTopic"][-1], + [ + "qri", + "inline-records", + "-p", + "part_of", + "-c", + "con-public", + ], + ) + self.assertIn("characterized_by", pages["xyzri:XYZDataset"][-1]) + self.assertEqual(homepage[0], ["qri", "list", "--pid", "xyzrins:."]) + self.assertTrue(all("con-public" in command for command in homepage[1:])) + + def test_projection_digest_scope_is_metadata_only(self) -> None: + specification = PROJECTION.load_yaml(PROJECTION.PROJECTION_SPEC_PATH) + specification["digest"]["scope"] = [ + "profiles/con/profile.yaml", + "profiles/con/projection.yaml", + "profiles/con/metadata", + "upstream:page_templates", + "upstream:code/pool2graph.py", + "upstream:.forgejo/workflows/update-from-pool.yaml", + ( + "parent:submodules/things-schemas/src/" + "demo-research-information/unreleased.yaml" + ), + "parent:tools/con_projection.py", + "component-commit-pins", + "projection-runtime-pins", + ] + labels = {label for label, _ in PROJECTION.input_files(specification)} + self.assertIn("parent/tools/con_projection.py", labels) + self.assertIn("upstream/code/pool2graph.py", labels) + self.assertIn( + "upstream/.forgejo/workflows/update-from-pool.yaml", + labels, + ) + self.assertTrue( + any(label.startswith("site/profiles/con/metadata/") for label in labels) + ) + self.assertFalse(any("editorial" in label for label in labels)) + self.assertFalse(any("assets" in label for label in labels)) + self.assertFalse(any("build_con_site" in label for label in labels)) + + runtime_pins = dict(PROJECTION.projection_runtime_pins()) + self.assertEqual(runtime_pins["linkml"], "==1.11.1") + self.assertEqual(runtime_pins["linkml-runtime"], "==1.11.1") + self.assertEqual(runtime_pins["pydantic"], "==2.13.4") + self.assertEqual(runtime_pins["rdflib"], "==7.6.0") + self.assertEqual(runtime_pins["packaging"], "==26.3") + self.assertEqual( + runtime_pins["local:dump-things-service"], + "path=submodules/dump-things-service", + ) + self.assertEqual( + runtime_pins["override:dump-things-pyclient"], + "path=submodules/dump-things-pyclient", + ) + runtime_records = PROJECTION.projection_runtime_lock_records() + runtime_labels = {label for label, _ in runtime_records} + self.assertTrue( + any(":pypi:pydantic-core@" in label for label in runtime_labels) + ) + self.assertFalse( + any( + excluded in label + for label in runtime_labels + for excluded in ( + ":conda:git-annex@", + ":conda:hugo@", + ":conda:libuv@", + ":conda:nodejs@", + ":pypi:pre-commit@", + ":pypi:snapper-fmt@", + ) + ) + ) + lock_digest = PROJECTION.projection_runtime_lock_digest() + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + with ( + patch.object(PROJECTION, "input_files", return_value=[]), + patch.object(PROJECTION, "projection_component_pins", return_value=[]), + patch.object(PROJECTION, "projection_runtime_pins", return_value=[]), + ): + manifest = PROJECTION.projection_manifest(output) + self.assertIn( + f"{lock_digest} pin:runtime-lock:projection-closure", + manifest, + ) + first_label, first_digest = runtime_records[0] + self.assertIn( + f"{first_digest} pin:runtime-resolved:{first_label}", + manifest, + ) + + projection_components = dict(PROJECTION.projection_component_pins()) + assembly_components = dict(PROJECTION.declared_component_pins()) + self.assertEqual( + set(projection_components), + { + "dump-things-pyclient", + "dump-things-service", + "query-things", + "things-schemas", + }, + ) + self.assertIn("congo", assembly_components) + self.assertIn("things-graph-renderer", assembly_components) + self.assertNotIn("congo", projection_components) + self.assertNotIn("things-graph-renderer", projection_components) + + def test_projection_local_runtime_paths_and_markers_fail_closed(self) -> None: + config = PROJECTION.tomllib.loads( + (ROOT / "pixi.toml").read_text(encoding="utf-8") + ) + pins = PROJECTION.projection_local_runtime_pins(config) + self.assertEqual( + pins["local:query-things"], + "path=submodules/query-things", + ) + for mutation in ("path", "extras", "override"): + with self.subTest(mutation=mutation): + candidate = deepcopy(config) + if mutation == "path": + candidate["pypi-dependencies"]["query-things"]["path"] = ( + "../alternate-query-things" + ) + elif mutation == "extras": + candidate["pypi-dependencies"]["dump-things-service"]["extras"] = [ + "unsafe" + ] + else: + candidate["pypi-options"]["dependency-overrides"][ + "dump-things-pyclient" + ]["path"] = "../alternate-client" + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "declared exactly", + ): + PROJECTION.projection_local_runtime_pins(candidate) + + environment = PROJECTION.lock_platform_environment( + "linux-64", + "linux-64", + "3.12.12", + ) + self.assertEqual(environment["platform_release"], "") + self.assertEqual(environment["platform_version"], "") + requirement = PROJECTION.Requirement( + "example; platform_release == 'host-specific'" + ) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "host-specific marker", + ): + PROJECTION.require_deterministic_marker(requirement) + + profile = PROJECTION.load_yaml(PROJECTION.PROFILE_PATH) + with tempfile.TemporaryDirectory() as directory: + directory_path = Path(directory) + + def profile_bytes(candidate: dict[str, object]) -> bytes: + path = directory_path / "profile.yaml" + path.write_text( + PROJECTION.yaml.safe_dump(candidate, sort_keys=False), + encoding="utf-8", + ) + return PROJECTION.projection_profile_digest_bytes(path) + + baseline_profile = profile_bytes(profile) + irrelevant_profile = deepcopy(profile) + irrelevant_profile["components"]["congo"]["commit"] = "0" * 40 + irrelevant_profile["components"]["graph"]["commit"] = "1" * 40 + self.assertEqual(profile_bytes(irrelevant_profile), baseline_profile) + relevant_profile = deepcopy(profile) + relevant_profile["identity"]["homepage_pid"] = "xyzrins:changed" + self.assertNotEqual(profile_bytes(relevant_profile), baseline_profile) + + def test_projection_lock_closure_ignores_unrelated_runtime_changes(self) -> None: + lock_path = ROOT / "pixi.lock" + document = PROJECTION.yaml.safe_load(lock_path.read_text(encoding="utf-8")) + baseline = PROJECTION.projection_runtime_lock_digest(lock_path) + + def changed_digest( + package_name: str | None = None, + conda_fragment: str | None = None, + ) -> str: + candidate = deepcopy(document) + matches = [ + package + for package in candidate["packages"] + if (package_name is not None and package.get("name") == package_name) + or ( + conda_fragment is not None + and conda_fragment in str(package.get("conda", "")) + ) + ] + self.assertTrue(matches) + matches[0]["sha256"] = "0" * 64 + with tempfile.TemporaryDirectory() as directory: + candidate_path = Path(directory) / "pixi.lock" + candidate_path.write_text( + PROJECTION.yaml.safe_dump(candidate, sort_keys=False), + encoding="utf-8", + ) + return PROJECTION.projection_runtime_lock_digest(candidate_path) + + for package_name, conda_fragment in ( + ("snapper-fmt", None), + (None, "/hugo-"), + (None, "/libuv-"), + ): + with self.subTest(irrelevant_package=package_name or conda_fragment): + self.assertEqual( + changed_digest(package_name, conda_fragment), + baseline, + ) + for package_name in ("pydantic", "markupsafe"): + with self.subTest(relevant_package=package_name): + self.assertNotEqual(changed_digest(package_name), baseline) + + def test_source_and_digest_symlink_escapes_are_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records = root / "records" + class_root = records / "XYZPerson" + class_root.mkdir(parents=True) + outside_record = root / "outside.yaml" + outside_record.write_text( + "pid: xyzrins:persons/outside\nschema_type: xyzri:XYZPerson\n", + encoding="utf-8", + ) + (class_root / "escaped.yaml").symlink_to(outside_record) + with self.assertRaisesRegex( + PROJECTION.ProjectionError, + "resolves outside", + ): + PROJECTION.source_records(records, "canonical") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + site = root / "site" + metadata = site / "profiles" / "con" / "metadata" + metadata.mkdir(parents=True) + outside = root / "outside.yaml" + outside.write_text("outside: true\n", encoding="utf-8") + (metadata / "escaped.yaml").symlink_to(outside) + specification = { + "digest": { + "scope": [ + "profiles/con/metadata", + "component-commit-pins", + "projection-runtime-pins", + ] + } + } + with ( + patch.object(PROJECTION, "SITE", site.resolve()), + self.assertRaisesRegex( + PROJECTION.ProjectionError, + "resolves outside", + ), + ): + PROJECTION.input_files(specification) + + def test_portrait_remains_an_annex_pointer_and_snapshot_has_no_assets( + self, + ) -> None: + portrait = "profiles/con/assets/img/yaroslav-halchenko.jpg" + entry = subprocess.run( + [ + "git", + "-C", + str(PROJECTION.SITE), + "ls-tree", + "HEAD", + portrait, + ], + check=True, + capture_output=True, + text=True, + ).stdout + self.assertTrue(entry.startswith("120000 blob ")) + target = subprocess.run( + [ + "git", + "-C", + str(PROJECTION.SITE), + "show", + f"HEAD:{portrait}", + ], + check=True, + capture_output=True, + text=True, + ).stdout + self.assertIn( + "MD5E-s37940--90e74fa17a709006dd527c5b36e41217.jpg", + target, + ) + assets = [ + path + for path in (PROJECTION.COMMITTED / "content").rglob("*") + if path.is_file() and path.suffix.lower() not in {".md"} + ] + self.assertEqual(assets, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_con_assembly.py b/tests/test_con_assembly.py new file mode 100644 index 0000000..5fc54e3 --- /dev/null +++ b/tests/test_con_assembly.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import importlib.util +import os +from pathlib import Path +import re +import sys +import tempfile +import tomllib +import unittest +from unittest import mock + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_tool(name: str): + path = ROOT / "tools" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +sys.path.insert(0, str(ROOT / "tools")) +BUILD = load_tool("build_con_site") + + +class CONAssemblyTests(unittest.TestCase): + def test_con_layout_overrides_are_preferred_digest_bound_and_static(self) -> None: + profile_layouts = BUILD.SITE / "profiles/con/layouts" + expected = { + Path("term.html"), + Path("_partials/article-link.html"), + Path("_partials/picture.html"), + Path("_partials/taxonomy-list-grid.html"), + Path("_partials/taxonomy-list-vertical-item.html"), + Path("_shortcodes/artwork-preview.html"), + } + actual = { + path.relative_to(profile_layouts) + for path in profile_layouts.rglob("*.html") + } + self.assertEqual(actual, expected) + + module = tomllib.loads( + (BUILD.SITE / "config/con/module.toml").read_text(encoding="utf-8") + ) + layout_sources = [ + mount["source"] + for mount in module["mounts"] + if mount.get("target") == "layouts" + ] + self.assertEqual(layout_sources, ["profiles/con/layouts", "layouts"]) + + params = tomllib.loads( + (BUILD.SITE / "config/con/params.toml").read_text(encoding="utf-8") + ) + self.assertIs(params.get("enableQuicklink"), False) + + assembly = BUILD.load_yaml(BUILD.ASSEMBLY_SPEC) + self.assertIn( + "profiles/con/layouts", + assembly["digest"]["scope"], + ) + + transformations = re.compile(r"\.(?:Resize|Fit|Fill|Crop|Process)\b") + for relative in sorted(expected): + text = (profile_layouts / relative).read_text(encoding="utf-8") + self.assertIsNone( + transformations.search(text), + f"CON layout invokes platform-dependent image processing: {relative}", + ) + + def test_quicklink_artifact_scan_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + site = Path(temporary) + (site / "index.html").write_text("
CON
\n", encoding="utf-8") + self.assertEqual(BUILD.quicklink_references(site), []) + + script = site / "js/main.js" + script.parent.mkdir() + script.write_text("quicklink.listen();\n", encoding="utf-8") + self.assertEqual(BUILD.quicklink_references(site), ["js/main.js"]) + + def test_manifest_tracks_site_parent_and_component_inputs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + site = root / "site" + profile = site / "profiles/con" + parent_input = root / "tools/builder.py" + editorial = profile / "editorial/content/about.md" + output = profile / "assembly/SHA256SUMS" + for path, content in ( + (parent_input, "builder\n"), + (editorial, "about\n"), + ): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + (profile / "profile.yaml").write_text( + yaml.safe_dump( + { + "paths": { + "assembly": "profiles/con/assembly.yaml", + "assembly_digest": ("profiles/con/assembly/SHA256SUMS"), + } + } + ), + encoding="utf-8", + ) + assembly = { + "digest": { + "algorithm": "sha256", + "output": "profiles/con/assembly/SHA256SUMS", + "scope": [ + "profiles/con/assembly.yaml", + "profiles/con/editorial/content", + "parent:tools/builder.py", + "component-commit-pins", + ], + } + } + spec_path = profile / "assembly.yaml" + spec_path.write_text(yaml.safe_dump(assembly), encoding="utf-8") + with ( + mock.patch.object(BUILD, "ROOT", root), + mock.patch.object(BUILD, "SITE", site), + mock.patch.object(BUILD, "PROFILE_ROOT", profile), + mock.patch.object(BUILD, "ASSEMBLY_SPEC", spec_path), + mock.patch.object( + BUILD, + "declared_component_pins", + return_value=[("site", "a" * 40)], + ), + mock.patch.object(BUILD, "verify_declared_pins"), + ): + first = BUILD.assembly_manifest() + BUILD.update_assembly_manifest() + BUILD.verify_assembly_manifest() + editorial.write_text("changed\n", encoding="utf-8") + second = BUILD.assembly_manifest() + self.assertNotEqual(first, second) + with self.assertRaisesRegex(BUILD.BuildError, "stale"): + BUILD.verify_assembly_manifest() + self.assertTrue(output.is_file()) + + def test_manifest_hashes_a_link_without_reading_its_target(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + outside = root / "outside.txt" + outside.write_text("secret-one\n", encoding="utf-8") + link = root / "asset.jpg" + link.symlink_to(outside) + first = BUILD.assembly_input_bytes(link) + outside.write_text("secret-two\n", encoding="utf-8") + self.assertEqual(first, BUILD.assembly_input_bytes(link)) + self.assertEqual(first, b"symlink\0" + os.readlink(link).encode()) + + def test_copy_sources_reject_undeclared_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "source" + source.mkdir() + (source / "outside").symlink_to(root) + with self.assertRaisesRegex(BUILD.BuildError, "directory symlink"): + BUILD.reject_source_symlinks(source) + + def test_generated_output_rejects_a_symlinked_ancestor(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + outside = root / "outside" + outside.mkdir() + linked = root / "site" / "profiles" + linked.parent.mkdir() + linked.symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(BUILD.BuildError, "symlinked ancestor"): + BUILD.reject_output_symlink_ancestors( + linked / "con/assembly/SHA256SUMS", + root / "site", + ) + + def test_manifest_update_ignores_a_predictable_temp_symlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + site = Path(temporary) / "site" + output = site / "profiles/con/assembly/SHA256SUMS" + output.parent.mkdir(parents=True) + outside = Path(temporary) / "outside.txt" + outside.write_text("unchanged\n", encoding="utf-8") + predictable = output.with_suffix(output.suffix + ".tmp") + predictable.symlink_to(outside) + with ( + mock.patch.object(BUILD, "SITE", site), + mock.patch.object( + BUILD, + "assembly_manifest_path", + return_value=output, + ), + mock.patch.object( + BUILD, + "assembly_manifest", + return_value="reviewed\n", + ), + ): + BUILD.update_assembly_manifest() + self.assertEqual(output.read_text(encoding="utf-8"), "reviewed\n") + self.assertTrue(predictable.is_symlink()) + self.assertEqual(outside.read_text(encoding="utf-8"), "unchanged\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_con_assets.py b/tests/test_con_assets.py new file mode 100644 index 0000000..557246a --- /dev/null +++ b/tests/test_con_assets.py @@ -0,0 +1,701 @@ +from __future__ import annotations + +import hashlib +import importlib.util +from io import BytesIO +import os +from pathlib import Path +from pathlib import PurePosixPath +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_tool(name: str): + path = ROOT / "tools" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +ASSETS = load_tool("con_assets") + + +PNG = b"\x89PNG\r\n\x1a\nfull-migration-test" +JPEG = b"\xff\xd8\xff\xe0full-migration-test" + + +def digest(payload: bytes, algorithm: str) -> str: + return hashlib.new(algorithm, payload).hexdigest() + + +def git(repository: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", repository, *arguments], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def canonical_pointer( + site: Path, + destination: Path, + key: str, + hashdir: PurePosixPath, +) -> str: + object_path = site / ".git/annex/objects" / Path(*hashdir.parts) / key / key + return Path(os.path.relpath(object_path, destination.parent)).as_posix() + + +def manifest() -> dict: + annex_key = f"MD5E-s{len(JPEG)}--{digest(JPEG, 'md5')}.jpg" + return { + "fallback_policy": { + "mode": "upstream-neutral", + "person": "meerkat-person", + "project": "meerkat-project", + "render_image": True, + }, + "omissions": { + "xyzrins:persons/missing": { + "kind": "portrait", + "availability": "unavailable", + "projection_link": None, + "fallback": "meerkat-person", + "source_path": "theme/static/img/team/missing.jpg", + "annex_key": "MD5E-s12--00000000000000000000000000000000.jpg", + "expected_size": 12, + }, + "xyzrins:projects/plain": { + "kind": "logo", + "availability": "absent-in-source", + "projection_link": None, + "fallback": "meerkat-project", + }, + }, + "assets": { + "profiles/con/assets/img/brand.png": { + "source_repository": "https://example.test/site.git", + "source_commit": "1" * 40, + "source_path": "theme/static/img/brand.png", + "availability": "available", + "storage": "git", + "media_type": "image/png", + "mode": "0644", + "size": len(PNG), + "sha256": digest(PNG, "sha256"), + "role": "site-brand", + }, + "profiles/con/assets/img/person.jpg": { + "source_repository": "https://example.test/site.git", + "source_commit": "1" * 40, + "source_path": "theme/static/img/person.jpg", + "availability": "available", + "storage": "git-annex", + "media_type": "image/jpeg", + "mode": "0644", + "size": len(JPEG), + "md5": digest(JPEG, "md5"), + "sha256": digest(JPEG, "sha256"), + "annex_key": annex_key, + "role": "person-portrait", + "retrieval": { + "remote": "example-read-only", + "repository": "https://assets.example.test/dataset/.git/", + "object_url": ( + "https://assets.example.test/dataset/.git/annex/" + f"objects/example/{annex_key}/{annex_key}" + ), + "mode": "read-only", + }, + }, + }, + "projection_links": { + "profiles/con/projection/content/persons/example/portrait.jpg": ( + "profiles/con/assets/img/person.jpg" + ), + }, + "static_links": { + "profiles/con/static/favicon.png": ("profiles/con/assets/img/brand.png"), + }, + } + + +class CONAssetTests(unittest.TestCase): + def test_all_declared_assets_hydrate_and_materialize(self) -> None: + declaration = manifest() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + site = root / "site" + cache = root / "cache" + site.mkdir() + git(site, "init", "--quiet") + brand = site / "profiles/con/assets/img/brand.png" + portrait = site / "profiles/con/assets/img/person.jpg" + brand.parent.mkdir(parents=True) + brand.write_bytes(PNG) + brand.chmod(0o644) + key = declaration["assets"]["profiles/con/assets/img/person.jpg"][ + "annex_key" + ] + hashdir = PurePosixPath("ab/cd") + portrait.symlink_to(canonical_pointer(site, portrait, key, hashdir)) + git(site, "add", "--all") + real_os_open = os.open + with ( + mock.patch.object(ASSETS, "SITE", site), + mock.patch.object(ASSETS, "CACHE", cache), + mock.patch.object( + ASSETS, + "annex_hashdir", + return_value=hashdir, + ), + mock.patch.object( + ASSETS, + "urlopen", + return_value=BytesIO(JPEG), + ), + mock.patch.object(ASSETS, "verify_annex_key") as annex_key, + mock.patch.object( + ASSETS.os, + "open", + wraps=real_os_open, + ) as secure_open, + ): + files = ASSETS.hydrate_manifest_assets(declaration) + assembly = root / "assembly" + materialized = ASSETS.materialize_all_assets( + assembly, + declaration, + files, + ) + + self.assertEqual(set(files), set(declaration["assets"])) + self.assertEqual(annex_key.call_count, 1) + hydrated_temporary = annex_key.call_args.args[0] + self.assertTrue(hydrated_temporary.name.startswith(".download-")) + self.assertTrue(hydrated_temporary.name.endswith("-person.jpg")) + temporary_flags = secure_open.call_args.args[1] + self.assertTrue(temporary_flags & os.O_EXCL) + self.assertTrue(temporary_flags & os.O_NOFOLLOW) + self.assertEqual(len(materialized), 4) + self.assertEqual( + ( + assembly + / "profiles/con/projection/content/persons/example/portrait.jpg" + ).read_bytes(), + JPEG, + ) + self.assertEqual( + (assembly / "profiles/con/static/favicon.png").read_bytes(), + PNG, + ) + self.assertTrue( + all(os.stat(path).st_mode & 0o777 == 0o644 for path in materialized) + ) + + def test_manifest_rejects_unsafe_or_incomplete_asset_contracts(self) -> None: + declaration = manifest() + bad_mode = { + **declaration, + "assets": { + **declaration["assets"], + "profiles/con/assets/img/brand.png": { + **declaration["assets"]["profiles/con/assets/img/brand.png"], + "mode": "0664", + }, + }, + } + with self.assertRaisesRegex(ASSETS.AssetError, "mode"): + ASSETS.asset_specs(bad_mode) + + traversal = { + **declaration, + "assets": { + "profiles/con/assets/../../secret.png": declaration["assets"][ + "profiles/con/assets/img/brand.png" + ] + }, + } + with self.assertRaisesRegex(ASSETS.AssetError, "normalized"): + ASSETS.asset_specs(traversal) + + writable = manifest() + writable["assets"]["profiles/con/assets/img/person.jpg"]["retrieval"][ + "mode" + ] = "write" + with self.assertRaisesRegex(ASSETS.AssetError, "read-only"): + ASSETS.asset_specs(writable) + + outside = manifest() + outside["assets"]["profiles/con/assets/img/person.jpg"]["retrieval"][ + "object_url" + ] = "https://elsewhere.example.test/object.jpg" + with self.assertRaisesRegex(ASSETS.AssetError, "outside"): + ASSETS.asset_specs(outside) + + missing_provenance = manifest() + del missing_provenance["assets"]["profiles/con/assets/img/brand.png"][ + "source_commit" + ] + with self.assertRaisesRegex(ASSETS.AssetError, "source_commit"): + ASSETS.asset_specs(missing_provenance) + + broken_fallback = manifest() + broken_fallback["fallback_policy"]["render_image"] = False + with self.assertRaisesRegex(ASSETS.AssetError, "fallback_policy"): + ASSETS.asset_specs(broken_fallback) + + linked_omission = manifest() + linked_omission["projection_links"][ + "profiles/con/projection/content/persons/missing/portrait.jpg" + ] = "profiles/con/assets/img/person.jpg" + with self.assertRaisesRegex(ASSETS.AssetError, "projection link"): + ASSETS.asset_specs(linked_omission) + + def test_every_repository_and_object_url_rejects_credentials(self) -> None: + for url in ( + "http://example.test/site.git", + "https://user@example.test/site.git", + "https://:secret@example.test/site.git", + "https://user:secret@example.test/site.git", + ): + with self.subTest(source_repository=url): + declaration = manifest() + declaration["assets"]["profiles/con/assets/img/brand.png"][ + "source_repository" + ] = url + with self.assertRaisesRegex(ASSETS.AssetError, "credential-free"): + ASSETS.asset_specs(declaration) + + for field in ("repository", "object_url"): + with self.subTest(retrieval_field=field): + declaration = manifest() + declaration["assets"]["profiles/con/assets/img/person.jpg"][ + "retrieval" + ][field] = "https://:secret@assets.example.test/dataset/.git/" + with self.assertRaisesRegex(ASSETS.AssetError, "credential-free"): + ASSETS.asset_specs(declaration) + + baseline = { + "website": { + "annex_metadata_commit": "a" * 40, + "upstream_repository": "https://user:secret@example.test/site.git", + } + } + with mock.patch.object(ASSETS, "load_yaml", return_value=baseline): + with self.assertRaisesRegex(ASSETS.AssetError, "credential-free"): + ASSETS.hydrate_upstream() + + with self.assertRaisesRegex(ASSETS.AssetError, "credential-free"): + ASSETS.annex_from_url( + Path("/tmp/example"), + "unsafe", + "https://user:secret@example.test/site.git", + "get", + action="Unsafe test transport", + ) + + def test_file_verification_checks_digest_mime_and_mode(self) -> None: + declaration = manifest() + spec = ASSETS.asset_specs(declaration)["profiles/con/assets/img/brand.png"] + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "brand.png" + path.write_bytes(PNG) + path.chmod(0o644) + ASSETS.verify_file(path, spec) + + path.chmod(0o600) + with self.assertRaisesRegex(ASSETS.AssetError, "mode"): + ASSETS.verify_file(path, spec) + path.chmod(0o644) + path.write_bytes(JPEG) + with self.assertRaisesRegex(ASSETS.AssetError, "bytes|sha256"): + ASSETS.verify_file(path, spec) + + wrong_mime = ASSETS.AssetSpec( + **{ + **spec.__dict__, + "size": len(PNG), + "sha256": digest(PNG, "sha256"), + "media_type": "image/jpeg", + } + ) + path.write_bytes(PNG) + with self.assertRaisesRegex(ASSETS.AssetError, "media type"): + ASSETS.verify_file(path, wrong_mime) + + def test_git_index_modes_and_annex_pointer_targets_are_exact(self) -> None: + declaration = manifest() + with tempfile.TemporaryDirectory() as temporary: + site = Path(temporary) / "site" + site.mkdir() + git(site, "init", "--quiet") + brand_relative = Path("profiles/con/assets/img/brand.png") + portrait_relative = Path("profiles/con/assets/img/person.jpg") + brand = site / brand_relative + portrait = site / portrait_relative + brand.parent.mkdir(parents=True) + brand.write_bytes(PNG) + brand.chmod(0o644) + key = declaration["assets"][portrait_relative.as_posix()]["annex_key"] + hashdir = PurePosixPath("ab/cd") + expected_pointer = canonical_pointer(site, portrait, key, hashdir) + portrait.symlink_to(expected_pointer) + git(site, "add", "--all") + + with ( + mock.patch.object(ASSETS, "SITE", site), + mock.patch.object( + ASSETS, + "annex_hashdir", + return_value=hashdir, + ), + ): + specs = ASSETS.asset_specs(declaration) + brand_spec = specs[brand_relative.as_posix()] + portrait_spec = specs[portrait_relative.as_posix()] + self.assertIsNone(ASSETS.verify_git_index_contract(brand_spec)) + self.assertEqual( + ASSETS.verify_git_index_contract(portrait_spec), + expected_pointer, + ) + ASSETS.verify_annex_pointer(portrait_spec, expected_pointer) + + git(site, "update-index", "--chmod=+x", brand_relative.as_posix()) + with self.assertRaisesRegex(ASSETS.AssetError, "100644"): + ASSETS.verify_git_index_contract(brand_spec) + git(site, "update-index", "--chmod=-x", brand_relative.as_posix()) + + portrait.unlink() + portrait.symlink_to("../../../../../wrong-annex-object") + with self.assertRaisesRegex(ASSETS.AssetError, "not canonical"): + ASSETS.verify_annex_pointer(portrait_spec, expected_pointer) + git(site, "add", "--", portrait_relative.as_posix()) + with self.assertRaisesRegex(ASSETS.AssetError, "not canonical"): + ASSETS.verify_git_index_contract(portrait_spec) + + def test_cache_rejects_symlinks_and_downloads_exclusively(self) -> None: + spec = ASSETS.asset_specs(manifest())["profiles/con/assets/img/person.jpg"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + cache = root / "cache" + outside = root / "outside" + cache.mkdir() + outside.mkdir() + (cache / "profiles").symlink_to(outside, target_is_directory=True) + with mock.patch.object(ASSETS, "CACHE", cache): + with self.assertRaisesRegex(ASSETS.AssetError, "ancestor is a symlink"): + ASSETS.hydrate_annex_asset(spec) + + safe_cache = root / "safe-cache" + destination = safe_cache.joinpath(*PurePosixPath(spec.destination).parts) + destination.parent.mkdir(parents=True) + outside_file = outside / "payload.jpg" + outside_file.write_bytes(JPEG) + destination.symlink_to(outside_file) + with mock.patch.object(ASSETS, "CACHE", safe_cache): + with self.assertRaisesRegex( + ASSETS.AssetError, + "cache destination is a symlink", + ): + ASSETS.hydrate_annex_asset(spec) + + collision = root / "exclusive-download" + collision.symlink_to(outside_file) + with self.assertRaisesRegex(ASSETS.AssetError, "exclusive"): + ASSETS.open_exclusive_download(collision) + + def test_materialization_rejects_symlinked_destination_ancestors(self) -> None: + spec = ASSETS.asset_specs(manifest())["profiles/con/assets/img/brand.png"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "source.png" + source.write_bytes(PNG) + source.chmod(0o644) + assembly = root / "assembly" + outside = root / "outside" + assembly.mkdir() + outside.mkdir() + (assembly / "profiles").symlink_to( + outside, + target_is_directory=True, + ) + with self.assertRaisesRegex(ASSETS.AssetError, "ancestor is a symlink"): + ASSETS.copy_materialized_file( + assembly, + spec.destination, + source, + spec, + ) + self.assertEqual(list(outside.iterdir()), []) + + def test_materialization_replaces_only_the_declared_annex_pointer(self) -> None: + spec = ASSETS.asset_specs(manifest())["profiles/con/assets/img/person.jpg"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + assembly = root / "assembly" + source = root / "person.jpg" + source.write_bytes(JPEG) + destination = assembly / spec.destination + destination.parent.mkdir(parents=True) + destination.symlink_to("../../canonical-annex-object") + with mock.patch.object( + ASSETS, + "canonical_annex_pointer_target", + return_value="../../canonical-annex-object", + ): + result = ASSETS.copy_materialized_file( + assembly, + spec.destination, + source, + spec, + ) + self.assertEqual(result.read_bytes(), JPEG) + self.assertFalse(result.is_symlink()) + + result.unlink() + result.symlink_to("../../different-object") + with ( + mock.patch.object( + ASSETS, + "canonical_annex_pointer_target", + return_value="../../canonical-annex-object", + ), + self.assertRaisesRegex(ASSETS.AssetError, "canonical annex pointer"), + ): + ASSETS.copy_materialized_file( + assembly, + spec.destination, + source, + spec, + ) + + def test_annex_pointer_is_independent_of_submodule_git_directory(self) -> None: + spec = ASSETS.asset_specs(manifest())["profiles/con/assets/img/person.jpg"] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + site = root / "site" + site.mkdir() + administrative_git_dir = root / "parent/.git/modules/site" + administrative_git_dir.mkdir(parents=True) + site.joinpath(".git").write_text( + f"gitdir: {administrative_git_dir}\n", + encoding="utf-8", + ) + hashdir = PurePosixPath("ab/cd") + destination = site / spec.destination + expected = canonical_pointer( + site, + destination, + spec.annex_key, + hashdir, + ) + with ( + mock.patch.object(ASSETS, "SITE", site), + mock.patch.object(ASSETS, "annex_hashdir", return_value=hashdir), + ): + actual = ASSETS.canonical_annex_pointer_target(spec) + + self.assertEqual(actual, expected) + self.assertNotIn(str(administrative_git_dir), actual) + + def test_runtime_and_annex_commands_are_pixi_scoped(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + baseline = root / "baseline.yaml" + baseline.write_text( + "toolchain:\n git_annex: '10.20260601'\n", + encoding="utf-8", + ) + executable = ROOT / ".pixi/envs/default/bin/git-annex" + + def command(arguments, **kwargs): + del kwargs + if any("import shutil" in argument for argument in arguments): + output = f"{executable}\n" + else: + output = "git-annex version: 10.20260601-gtest\n" + return mock.Mock(returncode=0, stdout=output, stderr="") + + with ( + mock.patch.object(ASSETS, "BASELINE_MANIFEST", baseline), + mock.patch.object(ASSETS, "run", side_effect=command) as run, + ): + ASSETS.verify_annex_runtime() + self.assertEqual(run.call_count, 2) + for call in run.call_args_list: + self.assertEqual(call.args[0][:2], ["pixi", "run"]) + + with mock.patch.object( + ASSETS, + "git", + return_value="/tmp/example.git", + ): + command = ASSETS.annex_command(Path("/tmp/example"), "version") + self.assertEqual(command[:3], ["pixi", "run", "git"]) + self.assertIn(f"user.name={ASSETS.ANNEX_BUILD_NAME}", command) + self.assertIn(f"user.email={ASSETS.ANNEX_BUILD_EMAIL}", command) + self.assertIn( + f"--work-tree={Path('/tmp/example').resolve()}", + command, + ) + self.assertNotIn("--global", command) + self.assertNotIn("core.worktree", " ".join(command)) + + def test_annex_payload_verification_supports_md5e_and_sha256e(self) -> None: + payloads = { + "MD5E": JPEG, + "SHA256E": PNG, + } + algorithms = { + "MD5E": "md5", + "SHA256E": "sha256", + } + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for backend, payload in payloads.items(): + with self.subTest(backend=backend): + path = root / f"{backend.lower()}.bin" + path.write_bytes(payload) + key = ( + f"{backend}-s{len(payload)}--" + f"{digest(payload, algorithms[backend])}.bin" + ) + with mock.patch.object( + ASSETS, + "run", + return_value=mock.Mock( + returncode=0, + stdout=f"{key}\n", + stderr="", + ), + ): + ASSETS.verify_payload_against_annex_key( + path, + key, + label=f"{backend} payload", + ) + path.write_bytes(b"x" * len(payload)) + with self.assertRaisesRegex(ASSETS.AssetError, "digest"): + ASSETS.verify_payload_against_annex_key( + path, + key, + label=f"{backend} payload", + ) + + def test_upstream_hydration_verifies_all_available_payloads(self) -> None: + md5_payload = b"a" + sha_payload = b"b" + md5_key = f"MD5E-s1--{digest(md5_payload, 'md5')}.txt" + sha_key = f"SHA256E-s1--{digest(sha_payload, 'sha256')}.txt" + entries = { + "assets/md5.txt": md5_key, + "static/sha.txt": sha_key, + } + baseline = { + "website": { + "annex_metadata_commit": "a" * 40, + "upstream_repository": "https://example.test/upstream.git", + } + } + with ( + mock.patch.object(ASSETS, "load_yaml", return_value=baseline), + mock.patch.object( + ASSETS, + "upstream_annex_entries", + return_value=entries, + ), + mock.patch.object( + ASSETS, + "annex", + side_effect=("", md5_key, sha_key), + ), + mock.patch.object( + ASSETS, + "annex_path_available", + side_effect=(True, True, True, True), + ), + mock.patch.object( + ASSETS, + "verify_payload_against_annex_key", + ) as verify_payload, + ): + ASSETS.hydrate_upstream() + self.assertEqual( + verify_payload.call_args_list, + [ + mock.call( + ASSETS.UPSTREAM / "assets/md5.txt", + md5_key, + label="Upstream annex payload assets/md5.txt", + ), + mock.call( + ASSETS.UPSTREAM / "static/sha.txt", + sha_key, + label="Upstream annex payload static/sha.txt", + ), + ], + ) + + def test_upstream_hydration_removes_inferred_remote_metadata(self) -> None: + baseline = { + "website": { + "annex_metadata_commit": "a" * 40, + "upstream_repository": "https://example.test/upstream.git", + } + } + key = "MD5E-s1--0cc175b9c0f1b6a831c399e269772661.txt" + with ( + mock.patch.object(ASSETS, "load_yaml", return_value=baseline), + mock.patch.object( + ASSETS, + "upstream_annex_entries", + return_value={"assets/example.txt": key}, + ), + mock.patch.object( + ASSETS, + "annex", + side_effect=("", key), + ), + mock.patch.object( + ASSETS, + "annex_path_available", + side_effect=(False, True), + ), + mock.patch.object(ASSETS, "temporary_remote_config", return_value=""), + mock.patch.object(ASSETS, "git"), + mock.patch.object(ASSETS, "annex_from_url"), + mock.patch.object( + ASSETS, + "remove_temporary_remote_config", + ) as remove, + mock.patch.object( + ASSETS, + "verify_payload_against_annex_key", + ) as verify_payload, + ): + ASSETS.hydrate_upstream() + remove.assert_called_once_with( + ASSETS.UPSTREAM, + "full-con-migration-upstream", + ) + verify_payload.assert_called_once_with( + ASSETS.UPSTREAM / "assets/example.txt", + key, + label="Upstream annex payload assets/example.txt", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_con_pages.py b/tests/test_con_pages.py new file mode 100644 index 0000000..4a00a90 --- /dev/null +++ b/tests/test_con_pages.py @@ -0,0 +1,445 @@ +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import sys +import tempfile +import tomllib +import unittest +from unittest import mock + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + + +def load_tool(name: str): + path = ROOT / "tools" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +PAGES = load_tool("build_con_pages") + + +def write_editor(root: Path) -> None: + root.mkdir(parents=True) + root.joinpath("index.html").write_text( + "Patch editor\n", + encoding="utf-8", + ) + root.joinpath("config.json").write_text( + json.dumps( + { + "class_url": "dlschemas_owl.ttl", + "data_url": "records.ttl", + "external_config_url": "config_default_xyzri.yaml", + "review_bundle_catalog": "record-sources.json", + "review_bundle_mode": "patch-download", + "shapes_url": "dlschemas_shacl.ttl", + "use_service": False, + "use_token": False, + }, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + for filename in ( + "config_default_xyzri.yaml", + "dlschemas_owl.ttl", + "dlschemas_shacl.ttl", + "records.ttl", + ): + root.joinpath(filename).write_text("# fixture\n", encoding="utf-8") + contract = { + "authentication": "none", + "backend": "none", + "mode": "patch-download", + "version": 1, + **PAGES.expected_editor_metadata(), + "input_sha256": PAGES.editor_input_digest(root), + } + root.joinpath("editor-contract.json").write_text( + json.dumps(contract, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def write_auditable_site(root: Path, *, local_url: bool = False) -> None: + root.mkdir(parents=True) + root.joinpath("index.html").write_text( + "CON\n" + '' + "Edit\n", + encoding="utf-8", + ) + root.joinpath(".nojekyll").write_bytes(b"") + editor = root / "edit" + write_editor(editor) + editor.joinpath("record-sources.json").write_text( + json.dumps(PAGES.canonical_record_catalog(), sort_keys=True) + "\n", + encoding="utf-8", + ) + if local_url: + root.joinpath("leak.js").write_text( + 'fetch("http://127.0.0.1:8111/api")\n', encoding="utf-8" + ) + + +class CONPagesTests(unittest.TestCase): + def test_pages_url_requires_credential_free_https(self) -> None: + self.assertEqual( + PAGES.normalized_pages_url("https://con.github.io/orinoco-lite-dev"), + ( + "https://con.github.io/orinoco-lite-dev/", + "/orinoco-lite-dev/", + ), + ) + for value in ( + "http://con.github.io/orinoco-lite-dev/", + "https://user@example.test/orinoco-lite-dev/", + "https://example.test/orinoco-lite-dev/?token=secret", + "https://example.test/../escape/", + ): + with self.subTest(value=value): + with self.assertRaises((PAGES.BuildError, ValueError)): + PAGES.normalized_pages_url(value) + + def test_editor_contract_rejects_service_mode_and_symlinks(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + editor = Path(temporary) / "editor" + write_editor(editor) + contract = editor / "editor-contract.json" + value = json.loads(contract.read_text(encoding="utf-8")) + value["backend"] = "dump-things" + contract.write_text(json.dumps(value), encoding="utf-8") + with self.assertRaisesRegex(PAGES.BuildError, "no backend"): + PAGES.validate_editor_source(editor) + + value["backend"] = "none" + contract.write_text(json.dumps(value), encoding="utf-8") + editor.joinpath("unsafe").symlink_to(Path(temporary)) + with self.assertRaisesRegex(PAGES.BuildError, "symlink"): + PAGES.validate_editor_source(editor) + + def test_editor_config_rejects_service_and_token_modes(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + editor = Path(temporary) / "editor" + write_editor(editor) + config_path = editor / "config.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["use_service"] = True + config_path.write_text(json.dumps(config), encoding="utf-8") + contract_path = editor / "editor-contract.json" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + contract["input_sha256"] = PAGES.editor_input_digest(editor) + contract_path.write_text(json.dumps(contract), encoding="utf-8") + with self.assertRaisesRegex(PAGES.BuildError, "disable service/token"): + PAGES.validate_editor_source(editor) + + def test_editor_config_rejects_remote_or_missing_static_inputs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + editor = Path(temporary) / "editor" + write_editor(editor) + config_path = editor / "config.json" + contract_path = editor / "editor-contract.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["data_url"] = "https://example.test/records.ttl" + config_path.write_text(json.dumps(config), encoding="utf-8") + contract = json.loads(contract_path.read_text(encoding="utf-8")) + contract["input_sha256"] = PAGES.editor_input_digest(editor) + contract_path.write_text(json.dumps(contract), encoding="utf-8") + with self.assertRaisesRegex(PAGES.BuildError, "normalized relative"): + PAGES.validate_editor_source(editor) + + config["data_url"] = "missing.ttl" + config_path.write_text(json.dumps(config), encoding="utf-8") + contract["input_sha256"] = PAGES.editor_input_digest(editor) + contract_path.write_text(json.dumps(contract), encoding="utf-8") + with self.assertRaisesRegex(PAGES.BuildError, "is missing"): + PAGES.validate_editor_source(editor) + + def test_editor_contract_rejects_stale_inputs_and_gitlinks(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + editor = Path(temporary) / "editor" + write_editor(editor) + editor.joinpath("index.html").write_text( + "changed\n", encoding="utf-8" + ) + with self.assertRaisesRegex(PAGES.BuildError, "digest is stale"): + PAGES.validate_editor_source(editor) + + write_editor(Path(temporary) / "fresh") + fresh = Path(temporary) / "fresh" + contract_path = fresh / "editor-contract.json" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + contract["site_commit"] = "0" * 40 + contract_path.write_text(json.dumps(contract), encoding="utf-8") + with self.assertRaisesRegex(PAGES.BuildError, "pinned pool UI"): + PAGES.validate_editor_source(fresh) + + def test_public_audit_rejects_loopback_service_urls(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "site" + write_auditable_site(destination, local_url=True) + with self.assertRaisesRegex(PAGES.BuildError, "local URL"): + PAGES.audit_pages_artifact( + destination, + PAGES.DEFAULT_BASE_URL, + require_editor=True, + ) + + def test_public_audit_rejects_symlinks_and_token_shaped_values(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "site" + write_auditable_site(destination) + destination.joinpath("outside").symlink_to(Path(temporary)) + destination.joinpath("secret.js").write_text( + 'const token = "ghp_abcdefghijklmnopqrstuvwxyz123456";\n', + encoding="utf-8", + ) + with self.assertRaises(PAGES.BuildError) as context: + PAGES.audit_pages_artifact( + destination, + PAGES.DEFAULT_BASE_URL, + require_editor=True, + ) + self.assertIn("public artifact symlink", str(context.exception)) + self.assertIn("GitHub token-shaped value", str(context.exception)) + + def test_public_audit_rejects_catalog_drift_and_custom_domain(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "site" + write_auditable_site(destination) + catalog_path = destination / "edit/record-sources.json" + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + catalog["records"] = [] + catalog_path.write_text(json.dumps(catalog), encoding="utf-8") + destination.joinpath("CNAME").write_text( + "www.centerforopenneuroscience.org\n", encoding="utf-8" + ) + with self.assertRaises(PAGES.BuildError) as context: + PAGES.audit_pages_artifact( + destination, + PAGES.DEFAULT_BASE_URL, + require_editor=True, + ) + self.assertIn("does not match canonical YAML", str(context.exception)) + self.assertIn("custom-domain", str(context.exception)) + + def test_publication_metadata_is_bound_to_payload_and_commits(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) / "site" + write_auditable_site(destination) + payload_entries, site_entries = PAGES.publication_manifest_entries( + destination + ) + publication = { + "base_path": "/orinoco-lite-dev/", + "base_url": PAGES.DEFAULT_BASE_URL, + "editor": "patch-download", + "files": len(payload_entries), + "parent_commit": PAGES.git_commit(ROOT), + "payload_manifest_sha256": PAGES.manifest_digest(payload_entries), + "site_commit": PAGES.git_commit(PAGES.SITE), + "site_manifest_sha256": PAGES.manifest_digest(site_entries), + "version": 1, + } + PAGES.write_json(destination / PAGES.PUBLICATION_NAME, publication) + self.assertEqual( + PAGES.publication_violations( + destination, + PAGES.DEFAULT_BASE_URL, + "patch-download", + ), + [], + ) + + publication["parent_commit"] = "0" * 40 + PAGES.write_json(destination / PAGES.PUBLICATION_NAME, publication) + self.assertIn( + "site: publication.json parent_commit is stale", + PAGES.publication_violations( + destination, + PAGES.DEFAULT_BASE_URL, + "patch-download", + ), + ) + + def test_build_embeds_editor_and_exact_record_catalog(self) -> None: + build_root = ROOT / "build" + build_root.mkdir(exist_ok=True) + with ( + tempfile.TemporaryDirectory(dir=build_root) as temporary, + tempfile.TemporaryDirectory() as editor_temporary, + ): + destination = Path(temporary) / "site" + editor = Path(editor_temporary) / "editor" + write_editor(editor) + + def build_site(path: Path, base_url: str): + self.assertEqual(base_url, PAGES.DEFAULT_BASE_URL) + self.assertEqual( + os.environ["SHACL_VUE_URL"], + "https://con.github.io/orinoco-lite-dev/edit/", + ) + path.mkdir(parents=True) + path.joinpath("index.html").write_text( + "CON\n" + 'Edit\n', + encoding="utf-8", + ) + entries = PAGES.manifest_entries(path) + return {"manifest_sha256": PAGES.manifest_digest(entries)} + + catalog = { + "format": "con-static-record-sources", + "records": [{"pid": "xyzrins:."}], + "site_commit": "b" * 40, + "version": 1, + } + with ( + mock.patch.object(PAGES, "build_site", side_effect=build_site), + mock.patch.object( + PAGES, "canonical_record_catalog", return_value=catalog + ), + ): + report = PAGES.build_pages_artifact( + destination, + PAGES.DEFAULT_BASE_URL, + editor_source=editor, + require_editor=True, + ) + + self.assertEqual(report["editor"], "patch-download") + self.assertTrue((destination / ".nojekyll").is_file()) + self.assertEqual( + json.loads( + (destination / "edit/record-sources.json").read_text( + encoding="utf-8" + ) + ), + catalog, + ) + publication = json.loads( + (destination / "publication.json").read_text(encoding="utf-8") + ) + _, site_entries = PAGES.publication_manifest_entries(destination) + self.assertEqual( + publication["site_manifest_sha256"], + PAGES.manifest_digest(site_entries), + ) + self.assertRegex(publication["payload_manifest_sha256"], r"^[0-9a-f]{64}$") + self.assertTrue( + destination.parent.joinpath("site-pages-manifest.sha256").is_file() + ) + + def test_pixi_pages_tasks_build_con_and_require_editor(self) -> None: + manifest = tomllib.loads((ROOT / "pixi.toml").read_text(encoding="utf-8")) + tasks = manifest["tasks"] + self.assertIn("build-pages-editor", tasks["build-pages"]["depends-on"]) + self.assertIn("verify-pages-editor", tasks["verify-pages"]["depends-on"]) + self.assertIn( + "build_con_pages.py --require-editor", tasks["build-pages"]["cmd"] + ) + self.assertIn("--repeat-destination", tasks["verify-pages"]["cmd"]) + self.assertNotIn("build_upstream_site", tasks["build-pages"]["cmd"]) + self.assertIn("build-pages", tasks["test-browser"]["depends-on"]) + self.assertIn("verify-pages", tasks["test-pages-browser"]["depends-on"]) + self.assertEqual( + tasks["test-pages-browser"]["env"]["PLAYWRIGHT_STATIC_ONLY"], + "1", + ) + + def test_workflow_never_deploys_pull_request_code(self) -> None: + workflow = ROOT / ".github/workflows/con-pages-preview.yml" + text = workflow.read_text(encoding="utf-8") + parsed = yaml.safe_load(text) + build = parsed["jobs"]["build"] + deploy = parsed["jobs"]["deploy"] + self.assertEqual(build["permissions"], {"contents": "read"}) + self.assertEqual( + deploy["permissions"], + {"contents": "read", "id-token": "write", "pages": "write"}, + ) + self.assertNotIn("pull_request", deploy["if"]) + self.assertFalse( + any( + str(step.get("uses", "")).startswith("actions/checkout@") + for step in deploy["steps"] + ) + ) + self.assertIn("pull_request:", text) + self.assertIn("pixi run test-pages-browser", text) + self.assertIn("persist-credentials: false", text) + self.assertIn("cache-write: ${{ github.event_name != 'pull_request' }}", text) + self.assertIn("submodules: recursive", text) + self.assertIn("include-hidden-files: true", text) + self.assertIn("github.event_name == 'push'", text) + self.assertNotIn("github.event_name == 'pull_request' ||", text) + for action in ( + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "prefix-dev/setup-pixi@f00437f565399d418b0acc85936d12c1fb668347", + "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9", + "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d", + "actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128", + ): + self.assertIn(action, text) + + def test_workflow_fetches_and_verifies_accepted_checkpoints(self) -> None: + workflow = yaml.safe_load( + (ROOT / ".github/workflows/con-pages-preview.yml").read_text( + encoding="utf-8" + ) + ) + steps = workflow["jobs"]["build"]["steps"] + names = [step["name"] for step in steps] + checkpoint_index = names.index( + "Verify the accepted clean-migration checkpoints" + ) + self.assertLess( + names.index("Check out the pinned recursive source tree"), + checkpoint_index, + ) + self.assertLess(checkpoint_index, names.index("Run focused contracts")) + + command = steps[checkpoint_index]["run"] + self.assertIn( + "parent_checkpoint=f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d", + command, + ) + self.assertIn( + "site_checkpoint=a122e506de9e4a13473edbe8d74a950d74032a16", + command, + ) + self.assertEqual( + command.count("refs/heads/codex/clean-migration:${checkpoint_ref}"), + 2, + ) + self.assertEqual(command.count("--no-recurse-submodules"), 2) + self.assertIn( + 'test "$(git rev-parse "${checkpoint_ref}")" = ' + '"${parent_checkpoint}"', + command, + ) + self.assertIn( + 'rev-parse "${checkpoint_ref}")" = "${site_checkpoint}"', + command, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_con_presentation.py b/tests/test_con_presentation.py new file mode 100644 index 0000000..3c754ce --- /dev/null +++ b/tests/test_con_presentation.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +TOOLS = ROOT / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +import build_con_site as BUILD # noqa: E402 +import con_projection as PROJECTION # noqa: E402 + + +class CONPresentationTests(unittest.TestCase): + def records(self) -> tuple[list[PROJECTION.SourceRecord], str]: + contract = PROJECTION.load_projection_contract() + return PROJECTION.source_closure(contract), contract.homepage_pid + + def test_contract_exactly_covers_people_projects_routes_and_menus(self) -> None: + records, homepage = self.records() + presentation = BUILD.validate_presentation_contract(records, homepage) + self.assertEqual(len(presentation.people), 33) + self.assertEqual(len(presentation.projects), 23) + self.assertEqual( + presentation.editorial_routes, + frozenset( + { + "about", + "contact", + "engage", + "explore", + "instruments", + "persons", + "projects", + "publications", + "support", + "whoweare", + } + ), + ) + self.assertEqual( + presentation.editorial_aliases, + frozenset( + { + "engage.html", + "projects.html", + "support.html", + "whoweare.html", + } + ), + ) + self.assertEqual( + BUILD.declared_taxonomy_routes(), + frozenset( + { + "datasets", + "instruments", + "objectives", + "persons", + "projects", + "publications", + "tags", + "topics", + } + ), + ) + + def test_duplicate_person_is_rejected(self) -> None: + records, homepage = self.records() + presentation = deepcopy(PROJECTION.load_yaml(BUILD.PRESENTATION)) + first = presentation["people"]["groups"][0]["members"][0] + presentation["people"]["groups"][1]["members"].append(first) + original_load = BUILD.load_yaml + + def load(path: Path): + if path == BUILD.PRESENTATION: + return presentation + return original_load(path) + + with mock.patch.object(BUILD, "load_yaml", side_effect=load): + with self.assertRaisesRegex(BUILD.BuildError, "not unique"): + BUILD.validate_presentation_contract(records, homepage) + + def test_person_group_boundary_is_executable(self) -> None: + records, homepage = self.records() + presentation = deepcopy(PROJECTION.load_yaml(BUILD.PRESENTATION)) + groups = presentation["people"]["groups"] + boundary_member = groups[0]["members"].pop() + groups[1]["members"].insert(0, boundary_member) + original_load = BUILD.load_yaml + + def load(path: Path): + if path == BUILD.PRESENTATION: + return presentation + return original_load(path) + + with mock.patch.object(BUILD, "load_yaml", side_effect=load): + with self.assertRaisesRegex(BUILD.BuildError, "groups/order"): + BUILD.validate_presentation_contract(records, homepage) + + def test_project_category_heading_is_executable(self) -> None: + records, homepage = self.records() + presentation = deepcopy(PROJECTION.load_yaml(BUILD.PRESENTATION)) + presentation["projects"]["categories"][0]["name"] = "Core software" + original_load = BUILD.load_yaml + + def load(path: Path): + if path == BUILD.PRESENTATION: + return presentation + return original_load(path) + + with mock.patch.object(BUILD, "load_yaml", side_effect=load): + with self.assertRaisesRegex(BUILD.BuildError, "categories/order"): + BUILD.validate_presentation_contract(records, homepage) + + def test_markdown_order_is_executable(self) -> None: + records, homepage = self.records() + presentation = deepcopy(PROJECTION.load_yaml(BUILD.PRESENTATION)) + members = presentation["people"]["groups"][0]["members"] + members[0], members[1] = members[1], members[0] + original_load = BUILD.load_yaml + + def load(path: Path): + if path == BUILD.PRESENTATION: + return presentation + return original_load(path) + + with mock.patch.object(BUILD, "load_yaml", side_effect=load): + with self.assertRaisesRegex(BUILD.BuildError, "editorial links"): + BUILD.validate_presentation_contract(records, homepage) + + def test_every_editorial_markdown_source_must_be_declared(self) -> None: + records, homepage = self.records() + presentation = deepcopy(PROJECTION.load_yaml(BUILD.PRESENTATION)) + presentation["editorial"]["routes"] = [ + route + for route in presentation["editorial"]["routes"] + if route["path"] != "/contact/" + ] + original_load = BUILD.load_yaml + + def load(path: Path): + if path == BUILD.PRESENTATION: + return presentation + return original_load(path) + + with mock.patch.object(BUILD, "load_yaml", side_effect=load): + with self.assertRaisesRegex(BUILD.BuildError, "source closure"): + BUILD.validate_presentation_contract(records, homepage) + + def test_published_html_routes_fail_closed(self) -> None: + entity_routes = {"persons/example"} + editorial_routes = {"about", "persons"} + with tempfile.TemporaryDirectory() as directory: + site = Path(directory) + (site / "index.html").write_text("home", encoding="utf-8") + for route in entity_routes | editorial_routes: + output = site / route / "index.html" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(route, encoding="utf-8") + + BUILD.verify_published_route_closure(site, entity_routes, editorial_routes) + + unexpected = site / "draft" / "index.html" + unexpected.parent.mkdir() + unexpected.write_text("draft", encoding="utf-8") + with self.assertRaisesRegex(BUILD.BuildError, "undeclared=.*draft"): + BUILD.verify_published_route_closure( + site, entity_routes, editorial_routes + ) + + unexpected.unlink() + alias = site / "unreviewed.html" + alias.write_text("alias", encoding="utf-8") + with self.assertRaisesRegex( + BUILD.BuildError, "undeclared=.*unreviewed.html" + ): + BUILD.verify_published_route_closure( + site, entity_routes, editorial_routes + ) + + alias.unlink() + (site / "persons" / "example" / "index.html").unlink() + with self.assertRaisesRegex(BUILD.BuildError, "missing=.*persons/example"): + BUILD.verify_published_route_closure( + site, entity_routes, editorial_routes + ) + + def test_menu_weight_drift_is_rejected(self) -> None: + records, homepage = self.records() + menu = BUILD.MENU_CONFIG.read_text(encoding="utf-8").replace( + "weight = 10", "weight = 11", 1 + ) + with tempfile.TemporaryDirectory() as directory: + menu_path = Path(directory) / "menus.en.toml" + menu_path.write_text(menu, encoding="utf-8") + with mock.patch.object(BUILD, "MENU_CONFIG", menu_path): + with self.assertRaisesRegex(BUILD.BuildError, "menus|menu"): + BUILD.validate_presentation_contract(records, homepage) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_local_stack_contract.py b/tests/test_local_stack_contract.py new file mode 100644 index 0000000..2d1fefa --- /dev/null +++ b/tests/test_local_stack_contract.py @@ -0,0 +1,126 @@ +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +POOL_UI = ROOT / "submodules" / "pool.psychoinformatics.de-ui" + + +class LocalStackContractTests(unittest.TestCase): + def test_pixi_exposes_all_local_services(self) -> None: + pixi = (ROOT / "pixi.toml").read_text(encoding="utf-8") + self.assertIn( + 'serve = { depends-on = ["build", "prepare-local-stack"], ' + 'cmd = "tools/serve_local_stack.sh" }', + pixi, + ) + self.assertIn( + 'prepare-local-stack = { depends-on = ["checkout-submodules", ' + '"build-pool-ui"], cmd = "python3 tools/prepare_local_stack.py" }', + pixi, + ) + self.assertIn( + 'refresh-local-pool = { depends-on = ["checkout-submodules", ' + '"build-pool-ui"], cmd = "REFRESH_UPSTREAM_POOL=1 python3 ' + 'tools/prepare_local_stack.py" }', + pixi, + ) + self.assertIn( + 'serve-shacl-vue = { depends-on = ["prepare-local-stack"], ' + 'cmd = "python3 -m http.server 3000 --directory ' + 'build/local-stack/ui" }', + pixi, + ) + self.assertIn( + 'build = { depends-on = ["checkout-submodules"], cmd = ' + '"python3 tools/build_con_site.py" }', + pixi, + ) + self.assertIn( + 'verify-static = { depends-on = ["checkout-submodules"], cmd = ' + '"python3 tools/build_con_site.py --repeat-destination ' + 'build/con-site-repeat" }', + pixi, + ) + self.assertIn( + 'serve-static = { depends-on = ["build"], cmd = ' + '"python3 -m http.server 8767 --directory build/con-site" }', + pixi, + ) + for task in ( + "build-upstream", + "serve-upstream", + "render-con-projection", + "update-con-projection", + "verify-con-projection", + "update-con-assembly", + "verify-con-assembly", + "prepare-local-stack", + "refresh-local-pool", + "serve-dump-things", + "serve-git-annex", + "seed-local-pool", + "serve-shacl-vue", + "check-local-stack", + ): + self.assertIn(f"{task} =", pixi) + + def test_pool_ui_points_at_local_upstream_services(self) -> None: + config = (POOL_UI / "config.yaml").read_text(encoding="utf-8") + self.assertIn("use_service: true", config) + self.assertIn("use_token: true", config) + self.assertIn("http://127.0.0.1:8111/protected/", config) + self.assertIn("http://127.0.0.1:8111/public/", config) + self.assertIn("http://127.0.0.1:8122/git-annex", config) + self.assertNotIn("https://hub.psychoinformatics.de/git-annex-p2phttp", config) + + external = (POOL_UI / "config_default_xyzri.yaml").read_text(encoding="utf-8") + self.assertIn("data_url: ''", external) + self.assertIn("get-record: 'record?pid={curie}&format=ttl'", external) + self.assertIn("xyzrins:", external) + + def test_schema_data_asset_is_not_a_demo_record_bundle(self) -> None: + data = (POOL_UI / "dlschemas_data.ttl").read_text(encoding="utf-8") + self.assertNotIn(" a xyzri:", data) + owl = (POOL_UI / "dlschemas_owl.ttl").read_text(encoding="utf-8") + self.assertIn("XYZDataset", owl) + + def test_pixi_pins_local_dump_things_runtime(self) -> None: + pixi = (ROOT / "pixi.toml").read_text(encoding="utf-8") + self.assertIn( + 'dump-things-service = { path = "submodules/dump-things-service" }', + pixi, + ) + for package, version in ( + ("linkml", "1.11.1"), + ("linkml-runtime", "1.11.1"), + ("pydantic", "2.13.4"), + ("rdflib", "7.6.0"), + ): + self.assertIn(f'{package} = "=={version}"', pixi) + + launcher = (ROOT / "tools" / "serve_local_dumpthings.sh").read_text( + encoding="utf-8" + ) + self.assertNotIn("uv run", launcher) + self.assertIn("exec dump-things-service", launcher) + + def test_annex_hydration_does_not_persist_transport_configuration(self) -> None: + assets = (ROOT / "tools" / "con_assets.py").read_text(encoding="utf-8") + self.assertNotIn('"remote",\n "add"', assets) + self.assertNotIn('"config",\n "core.worktree"', assets) + self.assertIn("annex_from_url", assets) + self.assertIn('"pixi",\n "run",\n "git",', assets) + self.assertIn('f"--work-tree={repository.resolve()}"', assets) + self.assertNotIn('["git-annex", "version"]', assets) + + builder = (ROOT / "tools" / "build_upstream_site.sh").read_text( + encoding="utf-8" + ) + self.assertNotIn("remote add", builder) + self.assertIn("restore_local_state", builder) + self.assertIn("--no-write-fetch-head", builder) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pages_editor.py b/tests/test_pages_editor.py new file mode 100644 index 0000000..40d84ba --- /dev/null +++ b/tests/test_pages_editor.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +from dump_things_service import Format +from dump_things_service.converter import FormatConverter + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + + +def load_tool(name: str): + path = ROOT / "tools" / f"{name}.py" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +BUILDER = load_tool("build_pages_editor") +APPLIER = load_tool("apply_editor_bundle") + + +class PagesEditorTests(unittest.TestCase): + def test_builder_emits_explicit_backend_free_contract(self) -> None: + build_root = ROOT / "build" + build_root.mkdir(exist_ok=True) + with ( + tempfile.TemporaryDirectory() as source_temporary, + tempfile.TemporaryDirectory(dir=build_root) as destination_temporary, + ): + source = Path(source_temporary) + source.joinpath("index.html").write_text( + "editor\n", encoding="utf-8" + ) + source.joinpath("config.yaml").write_text( + "use_service: true\n", encoding="utf-8" + ) + source.joinpath("bundle.js.map").write_text("{}\n", encoding="utf-8") + destination = Path(destination_temporary) / "editor" + with ( + mock.patch.object( + BUILDER, "static_records_turtle", return_value=("# records\n", 2) + ), + mock.patch.object(BUILDER, "git_commit", return_value="a" * 40), + ): + contract = BUILDER.build_editor(destination, source) + + config = json.loads( + destination.joinpath("config.json").read_text(encoding="utf-8") + ) + self.assertFalse(config["use_service"]) + self.assertFalse(config["use_token"]) + self.assertEqual(config["review_bundle_mode"], "patch-download") + self.assertEqual(config["review_bundle_catalog"], "record-sources.json") + self.assertFalse(destination.joinpath("config.yaml").exists()) + self.assertFalse(destination.joinpath("bundle.js.map").exists()) + self.assertEqual(contract["backend"], "none") + self.assertEqual(contract["authentication"], "none") + + def test_repeated_editor_builds_are_independent_and_byte_identical(self) -> None: + build_root = ROOT / "build" + build_root.mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=build_root) as temporary: + temporary_root = Path(temporary) + source = temporary_root / "source" + source.mkdir() + source.joinpath("index.html").write_text( + "editor\n", encoding="utf-8" + ) + first = temporary_root / "first" + second = temporary_root / "second" + build_number = 0 + + def build_pool_ui() -> None: + nonlocal build_number + build_number += 1 + source.joinpath("build.txt").write_text( + "deterministic\n", encoding="utf-8" + ) + + with ( + mock.patch.object(BUILDER, "DEFAULT_SOURCE", source), + mock.patch.object( + BUILDER, "static_records_turtle", return_value=("# records\n", 2) + ), + mock.patch.object(BUILDER, "git_commit", return_value="a" * 40), + mock.patch.object(BUILDER, "build_pool_ui", side_effect=build_pool_ui), + ): + report = BUILDER.verify_editor_builds(first, second, source) + + self.assertEqual(build_number, 2) + self.assertTrue(report["byte_identical"]) + self.assertEqual( + BUILDER.manifest_entries(first), + BUILDER.manifest_entries(second), + ) + + source.joinpath("build.txt").write_text("first\n", encoding="utf-8") + build_number = 0 + + def nondeterministic_build() -> None: + nonlocal build_number + build_number += 1 + source.joinpath("build.txt").write_text( + f"build {build_number}\n", encoding="utf-8" + ) + + with ( + mock.patch.object(BUILDER, "DEFAULT_SOURCE", source), + mock.patch.object( + BUILDER, "static_records_turtle", return_value=("# records\n", 2) + ), + mock.patch.object(BUILDER, "git_commit", return_value="a" * 40), + mock.patch.object( + BUILDER, "build_pool_ui", side_effect=nondeterministic_build + ), + ): + with self.assertRaisesRegex(BUILDER.BuildError, "byte-identical"): + BUILDER.verify_editor_builds(first, second, source) + + def test_editor_build_identity_does_not_depend_on_checkout_branch(self) -> None: + makefile = (BUILDER.UI / "Makefile").read_text(encoding="utf-8") + vite = (BUILDER.UI / "shacl-vue/vite.config.app.mjs").read_text( + encoding="utf-8" + ) + self.assertNotIn("rev-parse --abbrev-ref", makefile) + self.assertNotIn("rev-parse --abbrev-ref", vite) + self.assertIn('BUILD_GIT_BRANCH="pinned"', makefile) + self.assertIn("const branch = 'pinned';", vite) + self.assertEqual(makefile.count("rm -rf $(RUNTIME_PLUGIN_DIR)"), 2) + self.assertEqual(makefile.count("cp -r $(PLUGIN_DIR) $(RUNTIME_PLUGIN_DIR)"), 2) + + def test_static_rdf_is_canonical_across_blank_node_order(self) -> None: + first = """ + @prefix ex: . + ex:root ex:values [ ex:name "second" ], [ ex:name "first" ] . + """ + second = """ + @prefix ex: . + ex:root ex:values [ ex:name "first" ], [ ex:name "second" ] . + """ + + self.assertEqual( + BUILDER.canonical_turtle([first]), + BUILDER.canonical_turtle([second]), + ) + + def test_bundle_reader_rejects_extra_fields_and_oversized_input(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "bundle.json" + path.write_text( + json.dumps( + { + "format": APPLIER.FORMAT, + "records": [], + "site_commit": "a" * 40, + "unexpected": True, + "version": 1, + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(APPLIER.BuildError, "unexpected"): + APPLIER.read_bundle(path) + path.write_bytes(b"x" * (APPLIER.MAX_BUNDLE_BYTES + 1)) + with self.assertRaisesRegex(APPLIER.BuildError, "10 MiB"): + APPLIER.read_bundle(path) + + def test_applier_rejects_stale_commit_before_reading_sources(self) -> None: + bundle = { + "format": APPLIER.FORMAT, + "records": [{}], + "site_commit": "a" * 40, + "version": 1, + } + with ( + mock.patch.object(APPLIER, "require_clean_checkout"), + mock.patch.object(APPLIER, "git_commit", return_value="b" * 40), + ): + with self.assertRaisesRegex(APPLIER.BuildError, "stale"): + APPLIER.validate_bundle(bundle) + + def test_applier_rejects_a_dirty_checkout_and_uses_full_diff_paths(self) -> None: + with mock.patch.object( + APPLIER.subprocess, + "run", + return_value=mock.Mock(stdout="?? untracked.yaml\n"), + ): + with self.assertRaisesRegex(APPLIER.BuildError, "tracked or untracked"): + APPLIER.require_clean_checkout(APPLIER.SITE) + + with ( + mock.patch.object( + APPLIER.subprocess, "run", return_value=mock.Mock(stdout="") + ), + mock.patch.object( + APPLIER, + "projection_require_no_ignored_files", + side_effect=APPLIER.ProjectionError( + "The pinned static editor canonical inputs worktree has " + "ignored files" + ), + ), + ): + with self.assertRaisesRegex(APPLIER.BuildError, "ignored files"): + APPLIER.require_clean_checkout(APPLIER.SITE) + + source = ( + APPLIER.SITE + / "profiles/con/metadata/records/XYZPerson/yaroslav-halchenko.yaml" + ) + difference = APPLIER.diff_updates( + {source: source.read_text(encoding="utf-8") + "# changed\n"} + ) + self.assertIn( + "a/profiles/con/metadata/records/XYZPerson/yaroslav-halchenko.yaml", + difference, + ) + + def test_valid_bundle_is_bound_to_source_digest_and_path(self) -> None: + canonical, _, _ = APPLIER.canonical_index(APPLIER.SITE) + source = sorted(canonical.values(), key=lambda item: item.record["pid"])[0] + turtle = FormatConverter(str(APPLIER.SCHEMA), Format.json, Format.ttl).convert( + source.record, source.class_name + ) + content = source.path.read_bytes() + record = { + "pid": source.record["pid"], + "rdf_turtle": turtle, + "schema_type": source.record["schema_type"], + "source_path": source.path.relative_to(APPLIER.SITE).as_posix(), + "source_sha256": hashlib.sha256(content).hexdigest(), + } + bundle = { + "format": APPLIER.FORMAT, + "records": [record], + "site_commit": APPLIER.git_commit(APPLIER.SITE), + "version": 1, + } + updates = APPLIER.validate_bundle(bundle) + self.assertEqual(set(updates), {source.path}) + + stale = json.loads(json.dumps(bundle)) + stale["records"][0]["source_sha256"] = "0" * 64 + with self.assertRaisesRegex(APPLIER.BuildError, "digest is stale"): + APPLIER.validate_bundle(stale) + + escaped = json.loads(json.dumps(bundle)) + escaped["records"][0]["source_path"] = "../outside.yaml" + with self.assertRaisesRegex(APPLIER.BuildError, "path does not match"): + APPLIER.validate_bundle(escaped) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_playwright_contract.py b/tests/test_playwright_contract.py new file mode 100644 index 0000000..ef662fb --- /dev/null +++ b/tests/test_playwright_contract.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +from pathlib import Path +import tomllib +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class PlaywrightContractTests(unittest.TestCase): + def test_root_package_is_private_and_exactly_pinned(self) -> None: + package = json.loads((ROOT / "package.json").read_text(encoding="utf-8")) + self.assertTrue(package["private"]) + version = package["devDependencies"]["@playwright/test"] + self.assertRegex(version, r"^\d+\.\d+\.\d+$") + self.assertNotRegex(version, r"[~^*xX]") + + def test_browser_runtime_and_artifacts_stay_in_ignored_build_state(self) -> None: + pixi = (ROOT / "pixi.toml").read_text(encoding="utf-8") + self.assertIn('PLAYWRIGHT_BROWSERS_PATH = "build/playwright-browsers"', pixi) + self.assertIn("install-browser-tests =", pixi) + self.assertIn("install-pages-browser-tests =", pixi) + self.assertIn("test-browser =", pixi) + self.assertIn("test-pages-browser =", pixi) + tasks = tomllib.loads(pixi)["tasks"] + self.assertIn( + "install-browser-tests", + tasks["test-browser"]["depends-on"], + ) + config = (ROOT / "playwright.config.mjs").read_text(encoding="utf-8") + self.assertIn("build/playwright", config) + self.assertIn("reuseExistingServer: false", config) + self.assertIn("PLAYWRIGHT_STATIC_ONLY", config) + self.assertIn("workers: 1", config) + + def test_authenticated_spec_disables_secret_bearing_artifacts(self) -> None: + source = (ROOT / "tests/browser/authenticated-editor.spec.mjs").read_text( + encoding="utf-8" + ) + self.assertIn("trace: 'off'", source) + self.assertIn("screenshot: 'off'", source) + self.assertIn("video: 'off'", source) + self.assertNotIn("editor-token?", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/adapt_upstream_pages.py b/tools/adapt_upstream_pages.py new file mode 100755 index 0000000..9fcc5e8 --- /dev/null +++ b/tools/adapt_upstream_pages.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Adapt the upstream Hugo artifact for a GitHub Pages project path. + +Hugo applies ``baseURL`` to links it owns. The upstream Psychoinformatics +templates and graph bundle also contain a small number of root-absolute URLs +that Hugo cannot rewrite. This script adjusts only the generated artifact; +the pinned upstream source tree remains byte-for-byte unchanged. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from urllib.parse import unquote, urlsplit + + +HTML_URL_RE = re.compile( + r"(?P\b(?:href|src|action|poster)\s*=\s*)" + r"(?:" + r"(?P[\"'])(?P/[^\"']*)(?P=quote)" + r"|" + r"(?P/[^\s>]+)" + r")", + re.IGNORECASE, +) +GRAPH_FETCH_RE = re.compile( + r"(?P\bfetch\(\s*)(?P[\"'])" + r"(?P/[^\"']*graph\.json(?:\?[^\"']*)?)(?P=quote)" +) +GRAPH_SCRIPT_SRC_RE = re.compile( + r"(?P\bsrc\s*=\s*)" + r"(?:" + r"(?P[\"'])(?P/[^\"']*graph\.js(?:\?[^\"']*)?)(?P=quote)" + r"|" + r"(?P/[^\s>]*graph\.js(?:\?[^\s>]*)?)" + r")", + re.IGNORECASE, +) +EDIT_HREF_RE = re.compile( + r"(?P\bhref\s*=\s*)(?P[\"'])" + r"(?Phttps?://[^\"']+/ui/)", + re.IGNORECASE, +) +DEFAULT_EDIT_URL = "https://pool.psychoinformatics.de/ui/" + + +@dataclass +class AdaptationStats: + html_files_changed: int = 0 + html_urls_rewritten: int = 0 + graph_html_urls_versioned: int = 0 + graph_script_urls_rewritten: int = 0 + graph_node_urls_rewritten: int = 0 + webmanifest_urls_rewritten: int = 0 + edit_urls_rewritten: int = 0 + + +def normalize_base_path(value: str) -> str: + """Return a canonical root-relative path with leading/trailing slashes.""" + + value = unquote(value.strip()) + if not value.startswith("/"): + raise ValueError("base path must start with '/'") + if "?" in value or "#" in value or "\\" in value: + raise ValueError("base path cannot contain a query, fragment, or backslash") + + parts = [part for part in value.split("/") if part] + if any(part in {".", ".."} for part in parts): + raise ValueError("base path cannot contain '.' or '..' segments") + return "/" if not parts else f"/{'/'.join(parts)}/" + + +def normalize_edit_url(value: str) -> str: + """Return a URL suitable as the base of a SHACL Vue edit link.""" + + value = value.strip() + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("edit URL must be an absolute HTTP(S) URL") + if parsed.query or parsed.fragment: + raise ValueError("edit URL cannot contain a query or fragment") + return f"{value.rstrip('/')}/" + + +def prefix_root_url(url: str, base_path: str) -> str: + """Prefix a root-local URL unless it is external or already adapted.""" + + if not url.startswith("/") or url.startswith("//"): + return url + if base_path == "/": + return url + if url == base_path[:-1] or url.startswith(base_path): + return url + return f"{base_path}{url.lstrip('/')}" + + +def rewrite_html(text: str, base_path: str) -> tuple[str, int]: + rewrites = 0 + + def replace(match: re.Match[str]) -> str: + nonlocal rewrites + url = match.group("quoted") or match.group("bare") + rewritten = prefix_root_url(url, base_path) + if rewritten == url: + return match.group(0) + rewrites += 1 + quote = match.group("quote") or "" + return f"{match.group('prefix')}{quote}{rewritten}{quote}" + + return HTML_URL_RE.sub(replace, text), rewrites + + +def graph_resource_url(base_path: str, filename: str, bundle_key: str | None) -> str: + """Return the canonical root-local URL for a graph bundle resource.""" + + url = f"{base_path}{filename}" + return url if bundle_key is None else f"{url}?v={bundle_key}" + + +def rewrite_graph_script( + text: str, base_path: str, bundle_key: str | None = None +) -> tuple[str, int]: + """Normalize every graph data fetch to one optionally versioned URL.""" + + rewrites = 0 + expected_url = graph_resource_url(base_path, "graph.json", bundle_key) + + def replace(match: re.Match[str]) -> str: + nonlocal rewrites + if match.group("url") == expected_url: + return match.group(0) + rewrites += 1 + quote = match.group("quote") + return f"{match.group('prefix')}{quote}{expected_url}{quote}" + + return GRAPH_FETCH_RE.sub(replace, text), rewrites + + +def rewrite_graph_html_urls( + text: str, base_path: str, bundle_key: str +) -> tuple[str, int]: + """Give every generated graph script reference the bundle cache key.""" + + rewrites = 0 + expected_url = graph_resource_url(base_path, "graph.js", bundle_key) + + def replace(match: re.Match[str]) -> str: + nonlocal rewrites + current_url = match.group("quoted") or match.group("bare") + if current_url == expected_url: + return match.group(0) + rewrites += 1 + quote = match.group("quote") or "" + return f"{match.group('prefix')}{quote}{expected_url}{quote}" + + return GRAPH_SCRIPT_SRC_RE.sub(replace, text), rewrites + + +def graph_bundle_key(graph_script: str, graph_data: str) -> str: + """Hash a canonical manifest of the unversioned graph bundle bytes.""" + + entries = [] + for path, text in (("graph.js", graph_script), ("graph.json", graph_data)): + content = text.encode("utf-8") + entries.append( + { + "path": path, + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + ) + manifest = json.dumps( + {"files": entries, "version": 1}, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(manifest).hexdigest() + + +def expected_graph_bundle(site_dir: Path, base_path: str) -> tuple[str, str] | None: + """Return the expected cache key and unversioned script, if complete.""" + + graph_script_path = site_dir / "graph.js" + graph_data_path = site_dir / "graph.json" + if not graph_script_path.is_file() or not graph_data_path.is_file(): + return None + graph_script = graph_script_path.read_text(encoding="utf-8") + unversioned_script, _ = rewrite_graph_script(graph_script, base_path) + graph_data = graph_data_path.read_text(encoding="utf-8") + return graph_bundle_key(unversioned_script, graph_data), unversioned_script + + +def rewrite_edit_urls(text: str, edit_url: str) -> tuple[str, int]: + rewrites = 0 + + def replace(match: re.Match[str]) -> str: + nonlocal rewrites + current_url = match.group("url") + if current_url == edit_url: + return match.group(0) + rewrites += 1 + quote = match.group("quote") + return f"{match.group('prefix')}{quote}{edit_url}" + + return EDIT_HREF_RE.sub(replace, text), rewrites + + +def rewrite_graph_data(data: object, base_path: str) -> int: + if not isinstance(data, dict) or not isinstance(data.get("nodes"), list): + raise ValueError("graph.json does not contain a nodes list") + + rewrites = 0 + for node in data["nodes"]: + if not isinstance(node, dict) or not isinstance(node.get("url"), str): + continue + rewritten = prefix_root_url(node["url"], base_path) + if rewritten != node["url"]: + node["url"] = rewritten + rewrites += 1 + return rewrites + + +def rewrite_webmanifest_data(data: object, base_path: str) -> int: + """Prefix root-local icon URLs in a generated web app manifest.""" + + if not isinstance(data, dict) or not isinstance(data.get("icons"), list): + raise ValueError("site.webmanifest does not contain an icons list") + + rewrites = 0 + for icon in data["icons"]: + if not isinstance(icon, dict) or not isinstance(icon.get("src"), str): + continue + rewritten = prefix_root_url(icon["src"], base_path) + if rewritten != icon["src"]: + icon["src"] = rewritten + rewrites += 1 + return rewrites + + +def audit_site( + site_dir: Path, base_path: str, edit_url: str = DEFAULT_EDIT_URL +) -> list[str]: + """Return path, edit-link, and graph-bundle contract violations.""" + + base_path = normalize_base_path(base_path) + edit_url = normalize_edit_url(edit_url) + violations: list[str] = [] + graph_script_path = site_dir / "graph.js" + graph_data_path = site_dir / "graph.json" + graph_script_exists = graph_script_path.is_file() + graph_data_exists = graph_data_path.is_file() + bundle = expected_graph_bundle(site_dir, base_path) + expected_script_url: str | None = None + expected_data_url: str | None = None + if graph_script_exists != graph_data_exists: + missing = "graph.json" if graph_script_exists else "graph.js" + violations.append(f"site: graph bundle is missing {missing}") + if bundle is not None: + bundle_key, _ = bundle + expected_script_url = graph_resource_url(base_path, "graph.js", bundle_key) + expected_data_url = graph_resource_url(base_path, "graph.json", bundle_key) + + graph_html_references = 0 + for html_path in sorted(site_dir.rglob("*.html")): + text = html_path.read_text(encoding="utf-8") + for match in HTML_URL_RE.finditer(text): + url = match.group("quoted") or match.group("bare") + if prefix_root_url(url, base_path) != url: + violations.append(f"{html_path.relative_to(site_dir)}: {url}") + for match in EDIT_HREF_RE.finditer(text): + if match.group("url") != edit_url: + violations.append( + f"{html_path.relative_to(site_dir)}: edit URL {match.group('url')}" + ) + for match in GRAPH_SCRIPT_SRC_RE.finditer(text): + graph_html_references += 1 + url = match.group("quoted") or match.group("bare") + if expected_script_url is None: + violations.append( + f"{html_path.relative_to(site_dir)}: graph script URL {url} " + "has no complete graph bundle" + ) + elif url != expected_script_url: + violations.append( + f"{html_path.relative_to(site_dir)}: graph script URL {url} " + f"(expected {expected_script_url})" + ) + + if bundle is not None and graph_html_references == 0: + violations.append("site: graph.js has no HTML script reference") + + if graph_script_exists: + graph_script = graph_script_path.read_text(encoding="utf-8") + graph_fetches = list(GRAPH_FETCH_RE.finditer(graph_script)) + if not graph_fetches: + violations.append("graph.js: missing graph.json fetch") + elif expected_data_url is not None: + for match in graph_fetches: + if match.group("url") != expected_data_url: + violations.append( + f"graph.js: graph data URL {match.group('url')} " + f"(expected {expected_data_url})" + ) + + if graph_data_exists: + graph_data = json.loads(graph_data_path.read_text(encoding="utf-8")) + if not isinstance(graph_data, dict) or not isinstance( + graph_data.get("nodes"), list + ): + violations.append("graph.json: missing nodes list") + else: + for node in graph_data["nodes"]: + if not isinstance(node, dict) or not isinstance(node.get("url"), str): + continue + if prefix_root_url(node["url"], base_path) != node["url"]: + violations.append( + f"graph.json node {node.get('id', '')}: {node['url']}" + ) + + webmanifest_path = site_dir / "site.webmanifest" + if webmanifest_path.is_file(): + webmanifest = json.loads(webmanifest_path.read_text(encoding="utf-8")) + if not isinstance(webmanifest, dict) or not isinstance( + webmanifest.get("icons"), list + ): + violations.append("site.webmanifest: missing icons list") + else: + for index, icon in enumerate(webmanifest["icons"]): + if not isinstance(icon, dict) or not isinstance(icon.get("src"), str): + continue + if prefix_root_url(icon["src"], base_path) != icon["src"]: + violations.append(f"site.webmanifest icon {index}: {icon['src']}") + return violations + + +def adapt_site( + site_dir: Path, base_path: str, edit_url: str = DEFAULT_EDIT_URL +) -> AdaptationStats: + """Rewrite the generated site in place and fail if any path leaks remain.""" + + base_path = normalize_base_path(base_path) + edit_url = normalize_edit_url(edit_url) + if not site_dir.is_dir(): + raise FileNotFoundError(f"site directory does not exist: {site_dir}") + + stats = AdaptationStats() + + graph_data_path = site_dir / "graph.json" + if graph_data_path.is_file(): + graph_data = json.loads(graph_data_path.read_text(encoding="utf-8")) + count = rewrite_graph_data(graph_data, base_path) + if count: + graph_data_path.write_text( + json.dumps(graph_data, ensure_ascii=False, separators=(",", ":")) + + "\n", + encoding="utf-8", + ) + stats.graph_node_urls_rewritten += count + + graph_script_path = site_dir / "graph.js" + bundle = expected_graph_bundle(site_dir, base_path) + bundle_key: str | None = None + if bundle is not None: + bundle_key, _ = bundle + original = graph_script_path.read_text(encoding="utf-8") + rewritten, count = rewrite_graph_script(original, base_path, bundle_key) + if count: + graph_script_path.write_text(rewritten, encoding="utf-8") + stats.graph_script_urls_rewritten += count + + for html_path in sorted(site_dir.rglob("*.html")): + original = html_path.read_text(encoding="utf-8") + rewritten, count = rewrite_html(original, base_path) + rewritten, edit_count = rewrite_edit_urls(rewritten, edit_url) + graph_count = 0 + if bundle_key is not None: + rewritten, graph_count = rewrite_graph_html_urls( + rewritten, base_path, bundle_key + ) + if rewritten != original: + html_path.write_text(rewritten, encoding="utf-8") + stats.html_files_changed += 1 + stats.html_urls_rewritten += count + stats.edit_urls_rewritten += edit_count + stats.graph_html_urls_versioned += graph_count + + webmanifest_path = site_dir / "site.webmanifest" + if webmanifest_path.is_file(): + webmanifest = json.loads(webmanifest_path.read_text(encoding="utf-8")) + count = rewrite_webmanifest_data(webmanifest, base_path) + if count: + webmanifest_path.write_text( + json.dumps(webmanifest, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + stats.webmanifest_urls_rewritten += count + + violations = audit_site(site_dir, base_path, edit_url) + if violations: + sample = "\n".join(f" - {item}" for item in violations[:20]) + raise RuntimeError( + f"{len(violations)} root-path leak(s) remain after adaptation:\n{sample}" + ) + return stats + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("site_dir", type=Path, help="generated Hugo output directory") + parser.add_argument( + "--base-path", + required=True, + help="GitHub Pages base path, for example /orinoco-lite-dev", + ) + parser.add_argument( + "--check-only", + action="store_true", + help="report root-path leaks without changing the artifact", + ) + parser.add_argument( + "--edit-url", + default=DEFAULT_EDIT_URL, + help="SHACL Vue base URL for generated edit links", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + try: + base_path = normalize_base_path(args.base_path) + edit_url = normalize_edit_url(args.edit_url) + if not args.site_dir.is_dir(): + raise FileNotFoundError(f"site directory does not exist: {args.site_dir}") + if args.check_only: + violations = audit_site(args.site_dir, base_path, edit_url) + print( + json.dumps({"base_path": base_path, "violations": violations}, indent=2) + ) + return 1 if violations else 0 + + stats = adapt_site(args.site_dir, base_path, edit_url) + print(json.dumps({"base_path": base_path, **asdict(stats)}, indent=2)) + return 0 + except (FileNotFoundError, ValueError, RuntimeError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/apply_editor_bundle.py b/tools/apply_editor_bundle.py new file mode 100644 index 0000000..f31ab32 --- /dev/null +++ b/tools/apply_editor_bundle.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Validate a static editor review bundle and optionally update canonical YAML.""" + +from __future__ import annotations + +import argparse +from dataclasses import replace +import difflib +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import subprocess +import sys +import tempfile +from typing import Any, Sequence + +from dump_things_service import Format +from dump_things_service.converter import FormatConverter +import yaml + +from build_con_site import BuildError +from con_projection import ( + SCHEMA, + SITE, + ProjectionError, + SourceRecord, + load_projection_contract, + roundtrip_records, + source_closure, + require_no_ignored_files as projection_require_no_ignored_files, + validate_record_contract, +) + + +FORMAT = "con-shacl-review-bundle" +VERSION = 1 +MAX_BUNDLE_BYTES = 10 * 1024 * 1024 +MAX_RECORDS = 50 +TOP_LEVEL_KEYS = {"format", "records", "site_commit", "version"} +RECORD_KEYS = { + "pid", + "rdf_turtle", + "schema_type", + "source_path", + "source_sha256", +} + + +def git_commit(path: Path) -> str: + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if not re.fullmatch(r"[0-9a-f]{40}", result): + raise BuildError(f"Could not determine commit for {path}") + return result + + +def require_clean_checkout(path: Path) -> None: + status = subprocess.run( + ["git", "-C", str(path), "status", "--porcelain=v1", "--untracked-files=all"], + check=True, + capture_output=True, + text=True, + ).stdout + if status: + raise BuildError( + "Site checkout has tracked or untracked changes; preserve or commit " + "them before applying a review bundle" + ) + try: + projection_require_no_ignored_files( + path, + "static editor canonical inputs", + ( + "profiles/con/metadata", + "profiles/con/profile.yaml", + "profiles/con/projection.yaml", + ), + ) + except ProjectionError as error: + raise BuildError(str(error)) from error + + +def read_bundle(path: Path) -> dict[str, Any]: + if path.is_symlink() or not path.is_file(): + raise BuildError("Review bundle must be a regular file") + if path.stat().st_size > MAX_BUNDLE_BYTES: + raise BuildError("Review bundle is larger than 10 MiB") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise BuildError("Review bundle is not valid UTF-8 JSON") from error + if not isinstance(value, dict) or set(value) != TOP_LEVEL_KEYS: + raise BuildError("Review bundle has unexpected top-level fields") + records = value.get("records") + if ( + value.get("format") != FORMAT + or value.get("version") != VERSION + or not isinstance(records, list) + or not 0 < len(records) <= MAX_RECORDS + or not isinstance(value.get("site_commit"), str) + ): + raise BuildError("Review bundle does not satisfy version 1") + return value + + +def preserve_order(value: Any, template: Any) -> Any: + if isinstance(value, dict): + template_dict = template if isinstance(template, dict) else {} + ordered: dict[str, Any] = {} + for key in template_dict: + if key in value: + ordered[key] = preserve_order(value[key], template_dict[key]) + for key in sorted(set(value) - set(ordered)): + ordered[key] = preserve_order(value[key], None) + return ordered + if isinstance(value, list): + template_list = template if isinstance(template, list) else [] + return [ + preserve_order( + item, template_list[index] if index < len(template_list) else None + ) + for index, item in enumerate(value) + ] + return value + + +def dump_yaml(record: dict[str, Any], original: dict[str, Any]) -> str: + preferred = {"pid": record["pid"], "schema_type": record["schema_type"]} + preferred.update( + {key: value for key, value in record.items() if key not in preferred} + ) + ordered = preserve_order(preferred, {"pid": None, "schema_type": None, **original}) + return yaml.safe_dump( + ordered, + allow_unicode=True, + default_flow_style=False, + sort_keys=False, + width=80, + ) + + +def canonical_index( + site: Path, +) -> tuple[dict[str, SourceRecord], list[SourceRecord], Any]: + contract = load_projection_contract() + records = source_closure(contract) + canonical = { + record.record["pid"]: record + for record in records + if record.category == "canonical" + } + if not canonical: + raise BuildError("Canonical inventory is empty") + for record in canonical.values(): + if site.resolve() not in record.path.resolve().parents: + raise BuildError("Projection contract does not use the selected site") + return canonical, records, contract + + +def validate_bundle(bundle: dict[str, Any], site: Path = SITE) -> dict[Path, str]: + site = site.resolve() + require_clean_checkout(site) + if bundle["site_commit"] != git_commit(site): + raise BuildError("Review bundle is stale for the current site commit") + canonical, inventory, contract = canonical_index(site) + converter = FormatConverter(str(SCHEMA), Format.ttl, Format.json) + replacements: dict[str, SourceRecord] = {} + rendered: dict[Path, str] = {} + + for item in bundle["records"]: + if not isinstance(item, dict) or set(item) != RECORD_KEYS: + raise BuildError("Review bundle record has unexpected fields") + if not all(isinstance(item[key], str) for key in RECORD_KEYS): + raise BuildError("Review bundle record fields must be strings") + pid = item["pid"] + if pid in replacements: + raise BuildError(f"Review bundle contains duplicate PID: {pid}") + source = canonical.get(pid) + if source is None: + raise BuildError(f"Review bundle PID is not canonical: {pid}") + relative = PurePosixPath(item["source_path"]) + expected_relative = source.path.resolve().relative_to(site).as_posix() + if ( + relative.is_absolute() + or ".." in relative.parts + or relative.as_posix() != item["source_path"] + or item["source_path"] != expected_relative + ): + raise BuildError(f"Review bundle source path does not match {pid}") + if source.path.is_symlink() or not source.path.is_file(): + raise BuildError(f"Canonical source is not a regular file: {pid}") + original_bytes = source.path.read_bytes() + digest = hashlib.sha256(original_bytes).hexdigest() + if item["source_sha256"] != digest: + raise BuildError(f"Review bundle source digest is stale for {pid}") + if item["schema_type"] != source.record["schema_type"]: + raise BuildError(f"Review bundle schema type does not match {pid}") + if len(item["rdf_turtle"].encode("utf-8")) > 2 * 1024 * 1024: + raise BuildError(f"Review bundle RDF is too large for {pid}") + try: + restored = converter.convert(item["rdf_turtle"], source.class_name) + except Exception as error: + raise BuildError( + f"Review bundle RDF is invalid for {pid}: {error}" + ) from error + if not isinstance(restored, dict): + raise BuildError(f"Review bundle did not restore one record for {pid}") + restored["schema_type"] = item["schema_type"] + if restored.get("pid") != pid: + raise BuildError(f"Review bundle RDF changed the PID for {pid}") + replacements[pid] = replace(source, record=restored) + rendered[source.path] = dump_yaml(restored, source.record) + + candidate = [replacements.get(item.record["pid"], item) for item in inventory] + validate_record_contract(candidate, contract) + roundtrip_records(candidate) + return rendered + + +def diff_updates(updates: dict[Path, str], site: Path = SITE) -> str: + site = site.resolve() + chunks: list[str] = [] + for path, content in sorted(updates.items(), key=lambda item: str(item[0])): + before = path.read_text(encoding="utf-8") + relative = path.resolve().relative_to(site).as_posix() + chunks.extend( + difflib.unified_diff( + before.splitlines(keepends=True), + content.splitlines(keepends=True), + fromfile=f"a/{relative}", + tofile=f"b/{relative}", + ) + ) + return "".join(chunks) + + +def apply_updates(updates: dict[Path, str]) -> None: + for path, content in updates.items(): + if path.is_symlink() or not path.is_file(): + raise BuildError(f"Refusing to replace non-regular source: {path}") + mode = path.stat().st_mode + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=path.parent, delete=False + ) as temporary: + temporary.write(content) + temporary_path = Path(temporary.name) + try: + os.chmod(temporary_path, mode) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundle", type=Path) + parser.add_argument("--apply", action="store_true") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + updates = validate_bundle(read_bundle(args.bundle)) + difference = diff_updates(updates) + if not difference: + print("Review bundle is valid and produces no canonical YAML changes.") + return 0 + print(difference, end="") + if args.apply: + apply_updates(updates) + print(f"Applied {len(updates)} validated canonical record update(s).") + else: + print("Dry run only; rerun with --apply after reviewing this diff.") + return 0 + except (BuildError, ProjectionError, OSError, ValueError) as error: + print(f"CON editor bundle: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_con_pages.py b/tools/build_con_pages.py new file mode 100644 index 0000000..eda984d --- /dev/null +++ b/tools/build_con_pages.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +"""Build and audit the backend-free CON GitHub Pages artifact. + +The public artifact is deliberately self-contained. It embeds the canonical +record sources needed by a browser-only editor, but it never embeds a service +URL, credential, or GitHub write token. A separately built static editor can +be supplied with ``--editor-source``; without one, the artifact contains a +small read-only handoff page and the same deterministic record catalog. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import hashlib +import html +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import sys +from typing import Any, Iterator, Sequence +from urllib.parse import parse_qs, urlsplit + +from adapt_upstream_pages import audit_site, normalize_base_path +from build_con_site import ( + BuildError, + PROFILE_ROOT, + ROOT, + SITE, + build_site, + load_projection_contract, + load_yaml, + manifest_digest, + manifest_entries, + safe_destination, +) +from con_projection import ProjectionError, git_commit, source_closure + + +DEFAULT_DESTINATION = ROOT / "build" / "pages-preview" / "orinoco-lite-dev" +DEFAULT_REPEAT_DESTINATION = ( + ROOT / "build" / "pages-preview-repeat" / "orinoco-lite-dev" +) +DEFAULT_EDITOR_SOURCE = ROOT / "build" / "pages-editor" +DEFAULT_BASE_URL = "https://con.github.io/orinoco-lite-dev/" +EDITOR_ROUTE = "edit/" +CATALOG_NAME = "record-sources.json" +PUBLICATION_NAME = "publication.json" +PUBLICATION_KEYS = { + "base_path", + "base_url", + "editor", + "files", + "parent_commit", + "payload_manifest_sha256", + "site_commit", + "site_manifest_sha256", + "version", +} +POOL_UI = ROOT / "submodules" / "pool.psychoinformatics.de-ui" +THINGS_SCHEMAS = ROOT / "submodules" / "things-schemas" +LOCAL_URL_RE = re.compile( + rb"https?://(?:127(?:\.[0-9]{1,3}){3}|localhost)(?::[0-9]+)?", + re.IGNORECASE, +) +GERMAN_EDITOR_URL = b"https://pool.psychoinformatics.de/ui/" +GITHUB_TOKEN_RE = re.compile(rb"gh(?:[opusr]|pat)_[A-Za-z0-9_]{20,}") +EDIT_LINK_RE = re.compile( + r"\bhref\s*=\s*(?:\"(?P[^\"]*edit=true[^\"]*)\"|" + r"'(?P[^']*edit=true[^']*)'|(?P[^\s>]*edit=true[^\s>]*))", + re.IGNORECASE, +) + + +def normalized_pages_url(value: str) -> tuple[str, str]: + """Validate an HTTPS project URL and return it with its base path.""" + + try: + parsed = urlsplit(value.strip()) + parsed.port + except ValueError as error: + raise BuildError("Pages base URL is invalid") from error + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.hostname is None + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise BuildError("Pages base URL must be a credential-free absolute HTTPS URL") + base_path = normalize_base_path(parsed.path or "/") + return f"{parsed.scheme}://{parsed.netloc}{base_path}", base_path + + +def editor_url(base_url: str) -> str: + return f"{base_url.rstrip('/')}/{EDITOR_ROUTE}" + + +@contextmanager +def temporary_environment(name: str, value: str) -> Iterator[None]: + previous = os.environ.get(name) + os.environ[name] = value + try: + yield + finally: + if previous is None: + os.environ.pop(name, None) + else: + os.environ[name] = previous + + +def editor_input_digest(source: Path) -> str: + """Hash the editor inputs while excluding generated binding metadata.""" + + digest = hashlib.sha256() + excluded = {"editor-contract.json", CATALOG_NAME} + for path in sorted( + candidate for candidate in source.rglob("*") if candidate.is_file() + ): + relative = path.relative_to(source).as_posix() + if relative in excluded: + continue + digest.update( + relative.encode("utf-8") + + b"\0" + + hashlib.sha256(path.read_bytes()).digest() + ) + return digest.hexdigest() + + +def relative_editor_file(source: Path, value: Any, label: str) -> Path: + """Resolve one required editor-local file without accepting a URL.""" + + if not isinstance(value, str) or not value: + raise BuildError(f"Static editor {label} must be a relative file path") + parsed = urlsplit(value) + relative = PurePosixPath(parsed.path) + if ( + parsed.scheme + or parsed.netloc + or parsed.query + or parsed.fragment + or relative.is_absolute() + or relative.as_posix() != value + or any(part in {"", ".", ".."} for part in relative.parts) + ): + raise BuildError(f"Static editor {label} must be a normalized relative file") + candidate = source.joinpath(*relative.parts) + if candidate.is_symlink() or not candidate.is_file(): + raise BuildError(f"Static editor {label} is missing: {value}") + return candidate + + +def expected_editor_metadata() -> dict[str, Any]: + contract = load_projection_contract() + return { + "pool_ui_commit": git_commit(POOL_UI), + "record_count": len(source_closure(contract)), + "schema_commit": git_commit(THINGS_SCHEMAS), + "site_commit": git_commit(SITE), + } + + +def validate_editor_source(source: Path) -> dict[str, Any]: + """Validate the static, credential-free patch-download editor contract.""" + + if not source.is_dir() or not (source / "index.html").is_file(): + raise BuildError("Static editor source must contain index.html") + contract_path = source / "editor-contract.json" + config_path = source / "config.json" + if not contract_path.is_file() or not config_path.is_file(): + raise BuildError( + "Static editor source must contain editor-contract.json and config.json" + ) + try: + contract = json.loads(contract_path.read_text(encoding="utf-8")) + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise BuildError( + "Static editor contract/config is not valid UTF-8 JSON" + ) from error + expected = { + "authentication": "none", + "backend": "none", + "mode": "patch-download", + "version": 1, + } + if not isinstance(contract, dict) or any( + contract.get(key) != value for key, value in expected.items() + ): + raise BuildError( + "Static editor contract must declare version 1, patch-download " + "mode, no backend, and no authentication" + ) + expected_metadata = expected_editor_metadata() + if any(contract.get(key) != value for key, value in expected_metadata.items()): + raise BuildError( + "Static editor contract does not match the pinned pool UI, schema, " + "site, or record closure" + ) + if contract.get("input_sha256") != editor_input_digest(source): + raise BuildError("Static editor input digest is stale") + expected_config = { + "review_bundle_catalog": CATALOG_NAME, + "review_bundle_mode": "patch-download", + "use_service": False, + "use_token": False, + } + if not isinstance(config, dict) or any( + config.get(key) != value for key, value in expected_config.items() + ): + raise BuildError( + "Static editor config must disable service/token use and select " + "the relative patch-download record catalog" + ) + for key in ("class_url", "data_url", "external_config_url", "shapes_url"): + relative_editor_file(source, config.get(key), key) + forbidden_backend_fields = sorted( + key + for key, value in config.items() + if isinstance(key, str) + and key != "use_service" + and ("service_url" in key.lower() or key.lower() in {"api_url", "token"}) + and value is not None + and value != "" + and value is not False + ) + if forbidden_backend_fields: + raise BuildError( + "Static editor config retains backend/token fields: " + + ", ".join(forbidden_backend_fields) + ) + for candidate in source.rglob("*"): + relative = candidate.relative_to(source) + if candidate.is_symlink(): + raise BuildError(f"Static editor bundle contains a symlink: {relative}") + if ".git" in relative.parts: + raise BuildError(f"Static editor bundle contains Git state: {relative}") + return contract + + +def copy_editor(source: Path, destination: Path) -> None: + validate_editor_source(source) + shutil.copytree(source, destination, dirs_exist_ok=False) + + +def canonical_record_catalog() -> dict[str, Any]: + """Return exact canonical YAML inputs with immutable source coordinates.""" + + profile = load_yaml(PROFILE_ROOT / "profile.yaml") + contract = load_projection_contract(profile) + records = [] + for source in source_closure(contract): + if source.category != "canonical": + continue + resolved = source.path.resolve() + site_root = SITE.resolve() + if site_root not in resolved.parents: + raise BuildError( + f"Canonical record escapes the site checkout: {source.path}" + ) + relative = resolved.relative_to(site_root).as_posix() + if PurePosixPath(relative).as_posix() != relative: + raise BuildError(f"Canonical record path is not normalized: {relative}") + content = resolved.read_text(encoding="utf-8") + records.append( + { + "content": content, + "path": relative, + "pid": source.record["pid"], + "schema_type": source.record["schema_type"], + "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest(), + } + ) + records.sort(key=lambda item: (item["pid"], item["path"])) + return { + "format": "con-static-record-sources", + "patch_root": "centerforopenneuroscience.org", + "records": records, + "site_commit": git_commit(SITE), + "version": 1, + } + + +def write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def write_fallback_editor(destination: Path, base_path: str) -> None: + """Provide a truthful handoff when the optional editor is not bundled.""" + + destination.mkdir(parents=True, exist_ok=False) + catalog_url = f"{base_path}{EDITOR_ROUTE}{CATALOG_NAME}" + destination.joinpath("index.html").write_text( + "\n" + '\n' + '\n' + "CON metadata editing handoff\n" + "

Metadata editing handoff

\n" + "

This preview has no write service and stores no GitHub token. " + "The pinned canonical YAML sources are available for a local, " + "reviewable patch workflow.

\n" + f'

' + "Download the record source catalog

\n" + "

A browser editor can be added here once its patch-export bundle " + "passes the same static deployment checks.

\n" + "\n", + encoding="utf-8", + ) + + +def scan_public_artifact(destination: Path) -> list[str]: + """Return forbidden local/backend URLs found in public text assets.""" + + violations: list[str] = [] + text_suffixes = { + ".css", + ".html", + ".js", + ".json", + ".map", + ".md", + ".nt", + ".rdf", + ".svg", + ".toml", + ".ttl", + ".txt", + ".webmanifest", + ".xml", + ".yaml", + ".yml", + } + paths: list[Path] = [] + for directory, directory_names, file_names in os.walk( + destination, followlinks=False + ): + current = Path(directory) + kept_directories: list[str] = [] + for name in directory_names: + path = current / name + if path.is_symlink(): + violations.append( + f"{path.relative_to(destination)}: public artifact symlink" + ) + else: + kept_directories.append(name) + directory_names[:] = kept_directories + for name in file_names: + path = current / name + if path.is_symlink(): + violations.append( + f"{path.relative_to(destination)}: public artifact symlink" + ) + else: + paths.append(path) + for path in sorted(paths): + if not path.is_file() or path.suffix.lower() not in text_suffixes: + continue + content = path.read_bytes() + if match := LOCAL_URL_RE.search(content): + violations.append( + f"{path.relative_to(destination)}: local URL " + f"{match.group(0).decode('ascii', 'replace')}" + ) + if GERMAN_EDITOR_URL in content: + violations.append(f"{path.relative_to(destination)}: German editor URL") + if GITHUB_TOKEN_RE.search(content): + violations.append( + f"{path.relative_to(destination)}: GitHub token-shaped value" + ) + return violations + + +def audit_editor_links(destination: Path, expected_url: str) -> list[str]: + """Require every generated record edit link to use the static editor.""" + + expected = urlsplit(expected_url) + violations: list[str] = [] + count = 0 + for path in sorted(destination.rglob("*.html")): + text = path.read_text(encoding="utf-8") + for match in EDIT_LINK_RE.finditer(text): + count += 1 + value = html.unescape( + match.group("double") or match.group("single") or match.group("bare") + ) + parsed = urlsplit(value) + query = parse_qs(parsed.query, keep_blank_values=True) + if ( + (parsed.scheme, parsed.netloc, parsed.path) + != (expected.scheme, expected.netloc, expected.path) + or query.get("edit") != ["true"] + or not query.get("pid") + or not query.get("sh:NodeShape") + ): + violations.append( + f"{path.relative_to(destination)}: invalid static edit link" + ) + if count == 0: + violations.append("site: no record edit links target the static editor") + return violations + + +def manifest_path(entry: str) -> str: + try: + _, relative = entry.split(" ", 1) + except ValueError as error: + raise BuildError(f"Malformed artifact manifest entry: {entry}") from error + return relative + + +def publication_manifest_entries(destination: Path) -> tuple[list[str], list[str]]: + """Return the pre-publication payload and pre-editor site manifests.""" + + entries = [ + entry + for entry in manifest_entries(destination) + if manifest_path(entry) != PUBLICATION_NAME + ] + site_entries = [ + entry + for entry in entries + if manifest_path(entry) != ".nojekyll" + and not manifest_path(entry).startswith(EDITOR_ROUTE) + ] + return entries, site_entries + + +def publication_violations( + destination: Path, + base_url: str, + editor_kind: str, +) -> list[str]: + """Validate publication provenance against the exact current payload.""" + + path = destination / PUBLICATION_NAME + if path.is_symlink() or not path.is_file(): + return [f"site: missing {PUBLICATION_NAME}"] + try: + observed = json.loads(path.read_text(encoding="utf-8")) + payload_entries, site_entries = publication_manifest_entries(destination) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, BuildError) as error: + return [f"site: invalid {PUBLICATION_NAME}: {error}"] + if not isinstance(observed, dict) or set(observed) != PUBLICATION_KEYS: + return [f"site: {PUBLICATION_NAME} has unexpected fields"] + normalized_url, base_path = normalized_pages_url(base_url) + expected = { + "base_path": base_path, + "base_url": normalized_url, + "editor": editor_kind, + "files": len(payload_entries), + "parent_commit": git_commit(ROOT), + "payload_manifest_sha256": manifest_digest(payload_entries), + "site_commit": git_commit(SITE), + "site_manifest_sha256": manifest_digest(site_entries), + "version": 1, + } + return [ + f"site: {PUBLICATION_NAME} {key} is stale" + for key, value in expected.items() + if observed.get(key) != value + ] + + +def audit_pages_artifact( + destination: Path, + base_url: str, + *, + require_editor: bool = False, + require_publication: bool = True, +) -> dict[str, Any]: + base_url, base_path = normalized_pages_url(base_url) + expected_editor_url = editor_url(base_url) + violations = audit_site(destination, base_path, expected_editor_url) + violations.extend(scan_public_artifact(destination)) + violations.extend(audit_editor_links(destination, expected_editor_url)) + editor = destination / EDITOR_ROUTE + catalog = editor / CATALOG_NAME + if not (destination / ".nojekyll").is_file(): + violations.append("site: missing .nojekyll") + if not (editor / "index.html").is_file(): + violations.append("site: missing static editing handoff") + if not catalog.is_file(): + violations.append(f"site: missing {EDITOR_ROUTE}{CATALOG_NAME}") + else: + try: + observed_catalog = json.loads(catalog.read_text(encoding="utf-8")) + expected_catalog = canonical_record_catalog() + except (OSError, UnicodeDecodeError, json.JSONDecodeError, BuildError) as error: + violations.append(f"site: invalid static record catalog: {error}") + else: + if observed_catalog != expected_catalog: + violations.append( + "site: static record catalog does not match canonical YAML" + ) + if (destination / "CNAME").exists(): + violations.append("site: CNAME/custom-domain configuration is out of scope") + contract_path = editor / "editor-contract.json" + if contract_path.is_file(): + try: + validate_editor_source(editor) + except BuildError as error: + violations.append(f"site: {error}") + elif require_editor: + violations.append("site: required patch-export editor is not bundled") + editor_kind = "patch-download" if contract_path.is_file() else "record-handoff" + if require_publication: + violations.extend(publication_violations(destination, base_url, editor_kind)) + if violations: + detail = "\n".join(f" - {item}" for item in violations) + raise BuildError(f"Pages artifact audit failed:\n{detail}") + entries = manifest_entries(destination) + return { + "base_path": base_path, + "base_url": base_url, + "editor": editor_kind, + "files": len(entries), + "manifest_sha256": manifest_digest(entries), + } + + +def build_pages_artifact( + destination: Path, + base_url: str, + *, + editor_source: Path | None = None, + require_editor: bool = False, +) -> dict[str, Any]: + destination = safe_destination(destination) + base_url, base_path = normalized_pages_url(base_url) + if require_editor and editor_source is None: + raise BuildError("--require-editor requires --editor-source") + with temporary_environment("SHACL_VUE_URL", editor_url(base_url)): + site_report = build_site(destination, base_url) + + edit_destination = destination / EDITOR_ROUTE + if editor_source is None: + write_fallback_editor(edit_destination, base_path) + else: + copy_editor(editor_source.resolve(), edit_destination) + write_json(edit_destination / CATALOG_NAME, canonical_record_catalog()) + destination.joinpath(".nojekyll").write_bytes(b"") + payload_report = audit_pages_artifact( + destination, + base_url, + require_editor=require_editor, + require_publication=False, + ) + publication = { + **{ + key: value + for key, value in payload_report.items() + if key != "manifest_sha256" + }, + "parent_commit": git_commit(ROOT), + "payload_manifest_sha256": payload_report["manifest_sha256"], + "site_commit": git_commit(SITE), + "site_manifest_sha256": site_report["manifest_sha256"], + "version": 1, + } + write_json(destination / PUBLICATION_NAME, publication) + # Re-audit after publication metadata enters the uploaded artifact. + report = audit_pages_artifact( + destination, + base_url, + require_editor=require_editor, + ) + complete = {**publication, **report} + destination.parent.joinpath(f"{destination.name}-pages-build.json").write_text( + json.dumps(complete, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + destination.parent.joinpath(f"{destination.name}-pages-manifest.sha256").write_text( + "\n".join(manifest_entries(destination)) + "\n", + encoding="utf-8", + ) + return complete + + +def compare_pages_builds( + first: Path, + second: Path, + base_url: str, + *, + editor_source: Path | None = None, + require_editor: bool = False, +) -> dict[str, Any]: + first_report = build_pages_artifact( + first, + base_url, + editor_source=editor_source, + require_editor=require_editor, + ) + second_report = build_pages_artifact( + second, + base_url, + editor_source=editor_source, + require_editor=require_editor, + ) + if manifest_entries(first) != manifest_entries(second): + raise BuildError("Two clean Pages builds are not byte-identical") + return {"byte_identical": True, "first": first_report, "second": second_report} + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", type=Path, default=DEFAULT_DESTINATION) + parser.add_argument("--base-url", default=DEFAULT_BASE_URL) + parser.add_argument( + "--editor-source", + type=Path, + default=DEFAULT_EDITOR_SOURCE, + ) + parser.add_argument("--require-editor", action="store_true") + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--repeat-destination", type=Path) + args = parser.parse_args(argv) + try: + if args.check_only: + report = audit_pages_artifact( + args.destination, + args.base_url, + require_editor=args.require_editor, + ) + elif args.repeat_destination: + report = compare_pages_builds( + args.destination, + args.repeat_destination, + args.base_url, + editor_source=args.editor_source, + require_editor=args.require_editor, + ) + else: + report = build_pages_artifact( + args.destination, + args.base_url, + editor_source=args.editor_source, + require_editor=args.require_editor, + ) + print(json.dumps(report, sort_keys=True)) + except (BuildError, ProjectionError, OSError, ValueError) as error: + print(f"CON Pages build: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_con_site.py b/tools/build_con_site.py new file mode 100644 index 0000000..cc14006 --- /dev/null +++ b/tools/build_con_site.py @@ -0,0 +1,1211 @@ +#!/usr/bin/env python3 +"""Assemble and build the backend-free clean-migration website.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +from pathlib import PurePosixPath +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +from typing import Any, Sequence +from urllib.parse import urlsplit + +import yaml + +from con_assets import ( + ASSET_MANIFEST, + AssetError, + hydrate_all_assets, + load_yaml as load_asset_manifest, + materialize_all_assets, + upstream_annex_entries, +) +from con_projection import ( + BUILD_ROOT, + COMMITTED, + PROFILE_ROOT, + ProjectionExpectations, + ProjectionError, + ROOT, + SITE, + SourceRecord, + UPSTREAM, + declared_component_pins, + load_yaml, + load_projection_contract, + safe_reset, + source_closure, + stack_records, + validate_record_contract, + validate_projection, + verify_declared_pins, + verify_final_site_state, + verify_manifest, +) + + +ASSEMBLY = ROOT / "build" / "con-hugo" +ASSEMBLY_SPEC = PROFILE_ROOT / "assembly.yaml" +PRESENTATION = PROFILE_ROOT / "presentation.yaml" +MENU_CONFIG = SITE / "config" / "con" / "menus.en.toml" +TAXONOMY_CONFIG = SITE / "config" / "_default" / "taxonomies.toml" +DEFAULT_DESTINATION = ROOT / "build" / "con-site" +DEFAULT_BASE_URL = "http://127.0.0.1:8767/" +DEFAULT_EDIT_URL = "http://127.0.0.1:3000/" +ENTITY_SECTIONS = { + "datasets", + "instruments", + "objectives", + "organizations", + "persons", + "projects", + "publications", + "topics", +} +MARKDOWN_SUFFIXES = {".md", ".markdown", ".mdown"} + + +class BuildError(RuntimeError): + """Report a static assembly or site acceptance failure.""" + + +@dataclass(frozen=True) +class PresentationGroup: + """One named presentation group with its reviewed member order.""" + + name: str + members: tuple[str, ...] + + +@dataclass(frozen=True) +class PresentationContract: + """Reviewed navigation and ordering derived from canonical records.""" + + editorial_routes: frozenset[str] + editorial_aliases: frozenset[str] + people_groups: tuple[PresentationGroup, ...] + project_categories: tuple[PresentationGroup, ...] + people: tuple[str, ...] + projects: tuple[str, ...] + + +def run( + arguments: Sequence[str | Path], + *, + action: str, + environment: dict[str, str] | None = None, +) -> str: + result = subprocess.run( + [str(argument) for argument in arguments], + cwd=ROOT, + env=environment, + capture_output=True, + text=True, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + raise BuildError(f"{action} failed ({result.returncode}): {detail}") + return result.stdout + + +def safe_destination(path: Path) -> Path: + resolved = path.resolve() + build = (ROOT / "build").resolve() + temporary_roots = {Path("/tmp").resolve(), Path("/private/tmp").resolve()} + if build not in resolved.parents and not any( + temporary in resolved.parents for temporary in temporary_roots + ): + raise BuildError( + f"Destination must be below {build} or a temporary directory: {resolved}" + ) + return resolved + + +def copy_tree( + source: Path, + destination: Path, + *, + preserve_symlinks: bool = False, +) -> None: + if not source.is_dir(): + return + shutil.copytree( + source, + destination, + dirs_exist_ok=True, + symlinks=preserve_symlinks, + ignore=shutil.ignore_patterns(".git", ".DS_Store"), + ) + + +def source_symlinks(root: Path) -> list[Path]: + links: list[Path] = [] + for directory, names, filenames in os.walk(root, followlinks=False): + current = Path(directory) + kept: list[str] = [] + for name in names: + path = current / name + if path.is_symlink(): + raise BuildError(f"Source tree contains a directory symlink: {path}") + kept.append(name) + names[:] = kept + links.extend( + current / name for name in filenames if (current / name).is_symlink() + ) + return sorted(links) + + +def reject_source_symlinks(root: Path) -> None: + links = source_symlinks(root) + if links: + raise BuildError(f"Source tree contains an undeclared symlink: {links[0]}") + + +def validate_upstream_annex_symlinks(root: Path) -> None: + """Permit only exact, hydrated annex pointers recorded by provenance.""" + allowed = upstream_annex_entries() + for path in source_symlinks(root): + relative = path.relative_to(UPSTREAM).as_posix() + key = allowed.get(relative) + target = os.readlink(path).replace("\\", "/") + parts = PurePosixPath(target).parts + if ( + key is None + or len(parts) < 3 + or parts[-1] != key + or parts[-2] != key + or ".git/annex/objects/" not in target + or not path.resolve().is_file() + ): + raise BuildError(f"Unverified upstream symlink cannot be copied: {path}") + + +def assembly_scope_path(value: str) -> tuple[str, Path, Path]: + """Resolve one assembly input without following a nested symlink.""" + if value.startswith("upstream:"): + label, root, relative = "upstream", SITE, value.removeprefix("upstream:") + elif value.startswith("parent:"): + label, root, relative = "parent", ROOT, value.removeprefix("parent:") + else: + label, root, relative = "site", SITE, value + path = PurePosixPath(relative) + if ( + not relative + or relative.startswith("/") + or "\\" in relative + or ".." in path.parts + or path.as_posix() != relative + ): + raise BuildError(f"Invalid assembly digest scope path: {value}") + root = root.resolve() + candidate = root.joinpath(*path.parts) + if candidate != root and root not in candidate.parents: + raise BuildError(f"Assembly digest scope escapes {label}: {value}") + return label, root, candidate + + +def files_without_following_symlinks(root: Path) -> list[Path]: + """List regular files and link pointers while rejecting link directories.""" + if root.is_symlink(): + return [root] + if root.is_file(): + return [root] + if not root.is_dir(): + raise BuildError(f"Assembly digest input is absent: {root}") + files: list[Path] = [] + for directory, names, filenames in os.walk(root, followlinks=False): + current = Path(directory) + kept: list[str] = [] + for name in sorted(names): + path = current / name + if name in {".git", ".DS_Store"}: + continue + if path.is_symlink(): + raise BuildError( + f"Assembly digest input contains a directory symlink: {path}" + ) + kept.append(name) + names[:] = kept + for name in sorted(filenames): + if name in {".git", ".DS_Store"}: + continue + path = current / name + if not path.is_symlink() and not path.is_file(): + raise BuildError(f"Unsupported assembly digest input: {path}") + files.append(path) + return files + + +def assembly_input_bytes(path: Path) -> bytes: + if path.is_symlink(): + return b"symlink\0" + os.readlink(path).encode("utf-8") + return path.read_bytes() + + +def reject_output_symlink_ancestors(path: Path, root: Path) -> None: + """Reject a link or non-directory on the path to a generated output.""" + lexical_root = Path(os.path.abspath(root)) + lexical_path = Path(os.path.abspath(path)) + canonical_root = root.resolve() + for candidate_root in (lexical_root, canonical_root): + try: + relative = lexical_path.relative_to(candidate_root) + break + except ValueError: + continue + else: + raise BuildError(f"Generated output escapes its root: {path}") + root = canonical_root + current = root + for part in relative.parts[:-1]: + current /= part + if current.is_symlink(): + raise BuildError(f"Generated output has a symlinked ancestor: {current}") + if current.exists() and not current.is_dir(): + raise BuildError(f"Generated output ancestor is not a directory: {current}") + + +def assembly_manifest_path( + specification: dict[str, Any] | None = None, +) -> Path: + profile = load_yaml(PROFILE_ROOT / "profile.yaml") + paths = profile.get("paths") + if not isinstance(paths, dict): + raise BuildError("profiles/con/profile.yaml paths must be a mapping") + declared_spec = paths.get("assembly") + declared_output = paths.get("assembly_digest") + if not isinstance(declared_spec, str) or not isinstance(declared_output, str): + raise BuildError("The CON profile must declare assembly paths") + _, spec_root, spec_path = assembly_scope_path(declared_spec) + if spec_root != SITE.resolve() or spec_path != ASSEMBLY_SPEC.resolve(): + raise BuildError("The CON profile assembly path disagrees with the runtime") + specification = load_yaml(ASSEMBLY_SPEC) if specification is None else specification + digest = specification.get("digest") + if not isinstance(digest, dict) or digest.get("algorithm") != "sha256": + raise BuildError("profiles/con/assembly.yaml must use sha256") + output = digest.get("output") + if not isinstance(output, str): + raise BuildError("profiles/con/assembly.yaml must declare digest.output") + _, root, path = assembly_scope_path(output) + if root != SITE.resolve(): + raise BuildError("The assembly manifest output must be in the site checkout") + _, profile_root, profile_output = assembly_scope_path(declared_output) + if profile_root != SITE.resolve() or profile_output != path: + raise BuildError("The profile and assembly manifests disagree on digest output") + return path + + +def assembly_manifest() -> str: + """Describe every reviewed input that can change the static artifact.""" + verify_declared_pins(load_yaml(PROFILE_ROOT / "profile.yaml")) + specification = load_yaml(ASSEMBLY_SPEC) + digest = specification.get("digest") + if not isinstance(digest, dict): + raise BuildError("profiles/con/assembly.yaml digest must be a mapping") + scope = digest.get("scope") + if ( + not isinstance(scope, list) + or not scope + or not all(isinstance(item, str) and item for item in scope) + or len(scope) != len(set(scope)) + ): + raise BuildError("Assembly digest scope must be a unique string list") + if "component-commit-pins" not in scope: + raise BuildError("Assembly digest scope omits component-commit-pins") + output = assembly_manifest_path(specification) + entries: dict[str, Path] = {} + for item in scope: + if item == "component-commit-pins": + continue + label, root, path = assembly_scope_path(item) + for candidate in files_without_following_symlinks(path): + if candidate == output: + raise BuildError("Assembly digest output cannot be an input") + relative = candidate.relative_to(root).as_posix() + entry = f"{label}/{relative}" + if entry in entries: + raise BuildError(f"Assembly digest input is declared twice: {entry}") + entries[entry] = candidate + lines = ["# full-con-migration assembly manifest v1"] + for label, path in sorted(entries.items()): + lines.append( + f"{hashlib.sha256(assembly_input_bytes(path)).hexdigest()} input:{label}" + ) + for name, commit in declared_component_pins(): + digest_value = hashlib.sha256(f"{commit}\n".encode()).hexdigest() + lines.append(f"{digest_value} pin:{name}@{commit}") + return "\n".join([lines[0], *sorted(lines[1:])]) + "\n" + + +def verify_assembly_manifest() -> None: + path = assembly_manifest_path() + reject_output_symlink_ancestors(path, SITE) + if ( + path.is_symlink() + or not path.is_file() + or path.read_text(encoding="utf-8") != assembly_manifest() + ): + raise BuildError( + "The committed CON assembly digest is stale; run " + "`pixi run update-con-assembly` after reviewing site inputs" + ) + + +def update_assembly_manifest() -> Path: + path = assembly_manifest_path() + reject_output_symlink_ancestors(path, SITE) + if path.is_symlink(): + raise BuildError(f"Assembly manifest output is a symlink: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + text=True, + ) + temporary = Path(name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(assembly_manifest()) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if temporary.exists() or temporary.is_symlink(): + temporary.unlink() + return path + + +def remove_transport_overrides() -> None: + """Exclude upstream graph and branding that the CON profile replaces.""" + names = { + "android-chrome-192x192.png", + "android-chrome-512x512.png", + "apple-touch-icon.png", + "favicon-16x16.png", + "favicon-32x32.png", + "favicon.ico", + "graph.json", + "mstile-150x150.png", + "site.webmanifest", + } + roots = (ASSEMBLY / "static", ASSEMBLY / "themes" / "congo" / "static") + for root in roots: + for name in names: + path = root / name + if path.exists() or path.is_symlink(): + path.unlink() + for name in ("fzj.svg", "hhu.svg"): + path = ASSEMBLY / "assets" / "img" / name + if path.exists() or path.is_symlink(): + path.unlink() + + +def assemble_source( + asset_manifest: dict[str, Any], + asset_files: dict[str, Path], +) -> None: + safe_reset(ASSEMBLY) + for name in ("archetypes", "config", "layouts"): + reject_source_symlinks(SITE / name) + copy_tree(SITE / name, ASSEMBLY / name) + # The sibling checkout is only a hydration transport for unchanged annexed + # assets and the initialized theme. Its Git trees are checked against SITE. + for name in ("assets", "static", "themes"): + if name in {"assets", "static"}: + validate_upstream_annex_symlinks(UPSTREAM / name) + else: + reject_source_symlinks(UPSTREAM / name) + copy_tree(UPSTREAM / name, ASSEMBLY / name) + copy_tree(SITE / "config" / "con", ASSEMBLY / "config" / "con") + # Profile asset pointers are preserved for manifest-driven materialization, + # but a directory link must never redirect writes outside the assembly. + source_symlinks(PROFILE_ROOT) + copy_tree( + PROFILE_ROOT, + ASSEMBLY / "profiles" / "con", + preserve_symlinks=True, + ) + remove_transport_overrides() + materialize_all_assets(ASSEMBLY, asset_manifest, asset_files) + remaining_links = source_symlinks(ASSEMBLY) + if remaining_links: + raise BuildError( + f"Static assembly retains an undeclared symlink: {remaining_links[0]}" + ) + + +def artifact_asset_targets( + asset_manifest: dict[str, Any], +) -> dict[str, str]: + """Map each declared projected/static asset to its built-site path.""" + targets: dict[str, str] = {} + prefixes = { + "projection_links": "profiles/con/projection/content/", + "static_links": "profiles/con/static/", + } + for group, prefix in prefixes.items(): + links = asset_manifest.get(group, {}) + if not isinstance(links, dict): + raise BuildError(f"profiles/con/assets.yaml {group} is invalid") + for destination, source in sorted(links.items()): + if not isinstance(destination, str) or not isinstance(source, str): + raise BuildError("Asset links must be string mappings") + if not destination.startswith(prefix): + raise BuildError( + f"Asset destination is outside {prefix}: {destination}" + ) + relative = destination.removeprefix(prefix) + if not relative or relative in targets: + raise BuildError( + f"Asset destination does not map uniquely: {destination}" + ) + targets[relative] = source + return targets + + +def role_asset( + asset_manifest: dict[str, Any], + asset_files: dict[str, Path], + role: str, +) -> Path: + entries = asset_manifest.get("assets", {}) + if not isinstance(entries, dict): + raise BuildError("profiles/con/assets.yaml assets is invalid") + matches = [ + destination + for destination, entry in entries.items() + if isinstance(entry, dict) and entry.get("role") == role + ] + if len(matches) != 1 or matches[0] not in asset_files: + raise BuildError(f"Asset role must select one file: {role}") + return asset_files[matches[0]] + + +def presentation_groups( + section: dict[str, Any], + group_key: str, + member_key: str, + label: str, +) -> tuple[PresentationGroup, ...]: + groups = section.get(group_key) + if not isinstance(groups, list) or not groups: + raise BuildError(f"Presentation {label} must declare non-empty {group_key}") + names: list[str] = [] + result: list[PresentationGroup] = [] + all_members: list[str] = [] + for group in groups: + if not isinstance(group, dict): + raise BuildError(f"Presentation {label} group must be a mapping") + name = group.get("name") + values = group.get(member_key) + if ( + not isinstance(name, str) + or not name.strip() + or not isinstance(values, list) + or not values + or not all(isinstance(value, str) and value for value in values) + ): + raise BuildError(f"Presentation {label} group is invalid: {group!r}") + names.append(name) + members = tuple(values) + result.append(PresentationGroup(name, members)) + all_members.extend(members) + if len(names) != len(set(names)): + raise BuildError(f"Presentation {label} group names are not unique") + if len(all_members) != len(set(all_members)): + raise BuildError(f"Presentation {label} records are not unique") + return tuple(result) + + +def flattened_members(groups: Sequence[PresentationGroup]) -> tuple[str, ...]: + return tuple(member for group in groups for member in group.members) + + +def markdown_aliases(source: Path) -> frozenset[str]: + """Read safe, explicit HTML aliases from one editorial front matter block.""" + lines = source.read_text(encoding="utf-8").splitlines() + if not lines or lines[0].strip() != "---": + return frozenset() + try: + end = next( + index + for index, line in enumerate(lines[1:], start=1) + if line.strip() == "---" + ) + except StopIteration as error: + raise BuildError(f"Editorial front matter is unterminated: {source}") from error + metadata = yaml.safe_load("\n".join(lines[1:end])) or {} + aliases = metadata.get("aliases", []) if isinstance(metadata, dict) else None + if not isinstance(aliases, list) or not all( + isinstance(alias, str) + and re.fullmatch(r"/[a-z0-9][a-z0-9-]*\.html", alias) is not None + for alias in aliases + ): + raise BuildError(f"Editorial aliases are invalid: {source}") + if len(aliases) != len(set(aliases)): + raise BuildError(f"Editorial aliases are duplicated: {source}") + return frozenset(alias.removeprefix("/") for alias in aliases) + + +def presentation_routes( + presentation: dict[str, Any], +) -> tuple[dict[str, dict[str, Any]], frozenset[str], frozenset[str]]: + editorial = presentation.get("editorial") + routes = editorial.get("routes") if isinstance(editorial, dict) else None + if not isinstance(routes, list) or not routes: + raise BuildError("Presentation editorial.routes must be a non-empty list") + by_path: dict[str, dict[str, Any]] = {} + names: set[str] = set() + sources: set[str] = set() + weights: list[int] = [] + output_routes: set[str] = set() + output_aliases: set[str] = set() + source_root = PurePosixPath("profiles/con/editorial/content") + for entry in routes: + if not isinstance(entry, dict): + raise BuildError("Presentation editorial route must be a mapping") + name = entry.get("name") + path = entry.get("path") + source = entry.get("source") + navigation = entry.get("navigation") + weight = entry.get("weight") + if not isinstance(name, str) or not name.strip(): + raise BuildError(f"Presentation editorial route has no name: {entry!r}") + if ( + not isinstance(path, str) + or re.fullmatch(r"/[a-z0-9][a-z0-9/-]*/", path) is None + or "//" in path + ): + raise BuildError(f"Presentation editorial route is invalid: {path!r}") + if navigation not in {"main", "footer", "related"}: + raise BuildError( + f"Presentation editorial navigation is invalid: {navigation!r}" + ) + if not isinstance(weight, int) or isinstance(weight, bool) or weight <= 0: + raise BuildError(f"Presentation editorial weight is invalid: {weight!r}") + if not isinstance(source, str): + raise BuildError(f"Presentation editorial source is invalid: {source!r}") + source_path = PurePosixPath(source) + if ( + source_path.is_absolute() + or ".." in source_path.parts + or source_path.as_posix() != source + or not source_path.is_relative_to(source_root) + or source_path.suffix not in MARKDOWN_SUFFIXES + ): + raise BuildError( + f"Presentation editorial source escapes its root: {source}" + ) + resolved_source = SITE.joinpath(*source_path.parts) + if resolved_source.is_symlink() or not resolved_source.is_file(): + raise BuildError(f"Presentation editorial source is absent: {source}") + if path in by_path or name in names or source in sources: + raise BuildError(f"Presentation editorial routes are not unique: {entry!r}") + by_path[path] = entry + names.add(name) + sources.add(source) + weights.append(weight) + output_routes.add(path.strip("/")) + aliases = markdown_aliases(resolved_source) + overlap = output_aliases & aliases + if overlap: + raise BuildError( + f"Presentation editorial aliases are duplicated: {overlap}" + ) + output_aliases.update(aliases) + editorial_root = SITE.joinpath(*source_root.parts) + actual_sources = { + path.relative_to(SITE).as_posix() + for path in editorial_root.rglob("*") + if path.suffix in MARKDOWN_SUFFIXES and (path.is_file() or path.is_symlink()) + } + if sources != actual_sources: + raise BuildError( + "Presentation editorial source closure disagrees with the checkout: " + f"undeclared={sorted(actual_sources - sources)}, " + f"absent={sorted(sources - actual_sources)}" + ) + if len(weights) != len(set(weights)) or weights != sorted(weights): + raise BuildError("Presentation editorial weights must be unique and ordered") + return by_path, frozenset(output_routes), frozenset(output_aliases) + + +def ordered_editorial_groups( + source: str, + section: str, +) -> tuple[PresentationGroup, ...]: + """Read exact level-two headings and entity-link order from Markdown.""" + heading_pattern = re.compile(r"##[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*") + reference_pattern = re.compile( + r'\{\{<\s*ref\s+"(/' + re.escape(section) + r'/[^"/]+)"\s*>\}\}' + ) + groups: list[PresentationGroup] = [] + name: str | None = None + members: list[str] = [] + for line in (SITE / source).read_text(encoding="utf-8").splitlines(): + heading = heading_pattern.fullmatch(line) + if heading: + if name is not None: + groups.append(PresentationGroup(name, tuple(members))) + name = heading.group(1) + members = [] + references = reference_pattern.findall(line) + if references and name is None: + raise BuildError( + f"Editorial {section} link appears before its group heading" + ) + members.extend(reference.removeprefix("/") for reference in references) + if name is not None: + groups.append(PresentationGroup(name, tuple(members))) + return tuple(groups) + + +def ordered_editorial_refs(source: str, section: str) -> tuple[str, ...]: + return flattened_members(ordered_editorial_groups(source, section)) + + +def validate_presentation_contract( + records: Sequence[SourceRecord], + homepage_pid: str, +) -> PresentationContract: + """Require presentation groups, routes, menus, and canonical data to agree.""" + profile = load_yaml(PROFILE_ROOT / "profile.yaml") + paths = profile.get("paths") + if not isinstance(paths, dict) or paths.get("presentation") != ( + PRESENTATION.relative_to(SITE).as_posix() + ): + raise BuildError("The profile presentation path disagrees with the runtime") + presentation = load_yaml(PRESENTATION) + if presentation.get("version") != 1 or presentation.get("profile") != "con": + raise BuildError("profiles/con/presentation.yaml has an unsupported identity") + + people_section = presentation.get("people") + projects_section = presentation.get("projects") + if not isinstance(people_section, dict) or not isinstance(projects_section, dict): + raise BuildError("Presentation people/projects must be mappings") + people_groups = presentation_groups(people_section, "groups", "members", "people") + project_categories = presentation_groups( + projects_section, "categories", "projects", "projects" + ) + people = flattened_members(people_groups) + projects = flattened_members(project_categories) + + canonical_people = { + record.record["pid"] + for record in records + if record.category == "canonical" + and record.record.get("schema_type") == "xyzri:XYZPerson" + } + canonical_projects = { + record.record["pid"] + for record in records + if record.category == "canonical" + and record.record.get("schema_type") == "xyzri:XYZProject" + and record.record["pid"] != homepage_pid + } + if set(people) != canonical_people: + raise BuildError("Presentation people do not exactly cover canonical people") + if set(projects) != canonical_projects: + raise BuildError( + "Presentation projects do not exactly cover canonical projects" + ) + + by_path, editorial_routes, editorial_aliases = presentation_routes(presentation) + people_route = people_section.get("route") + projects_route = projects_section.get("route") + if people_route not in by_path or projects_route not in by_path: + raise BuildError("Presentation group landing routes are not editorial routes") + expected_people_groups = tuple( + PresentationGroup( + group.name, + tuple(pid.removeprefix("xyzrins:") for pid in group.members), + ) + for group in people_groups + ) + expected_project_categories = tuple( + PresentationGroup( + category.name, + tuple(pid.removeprefix("xyzrins:") for pid in category.members), + ) + for category in project_categories + ) + actual_people_groups = ordered_editorial_groups( + by_path[people_route]["source"], "persons" + ) + actual_project_categories = ordered_editorial_groups( + by_path[projects_route]["source"], "projects" + ) + if actual_people_groups != expected_people_groups: + raise BuildError( + "People editorial links/headings disagree with presentation groups/order" + ) + if actual_project_categories != expected_project_categories: + raise BuildError( + "Project editorial links/headings disagree with presentation " + "categories/order" + ) + + menu = tomllib.loads(MENU_CONFIG.read_text(encoding="utf-8")) + declared_menu: dict[tuple[str, str], int] = {} + for path, entry in by_path.items(): + navigation = entry["navigation"] + if navigation in {"main", "footer"}: + declared_menu[(navigation, path.strip("/"))] = entry["weight"] + actual_menu: dict[tuple[str, str], int] = {} + for navigation in ("main", "footer"): + entries = menu.get(navigation, []) + if not isinstance(entries, list): + raise BuildError(f"CON {navigation} menu must be an array") + for entry in entries: + if not isinstance(entry, dict): + raise BuildError(f"CON {navigation} menu entry must be a mapping") + page_ref = entry.get("pageRef") + weight = entry.get("weight") + if not isinstance(page_ref, str) or not isinstance(weight, int): + raise BuildError(f"CON {navigation} menu entry is invalid: {entry!r}") + key = (navigation, page_ref.strip("/")) + if key in actual_menu: + raise BuildError(f"CON menu route is duplicated: {key}") + actual_menu[key] = weight + if actual_menu != declared_menu: + raise BuildError("CON menu routes/weights disagree with presentation.yaml") + + return PresentationContract( + editorial_routes=editorial_routes, + editorial_aliases=editorial_aliases, + people_groups=people_groups, + project_categories=project_categories, + people=people, + projects=projects, + ) + + +def manifest_entries(root: Path) -> list[str]: + return [ + f"{hashlib.sha256(path.read_bytes()).hexdigest()} " + f"{path.relative_to(root).as_posix()}" + for path in sorted(root.rglob("*")) + if path.is_file() and path.name != ".DS_Store" + ] + + +def manifest_digest(entries: list[str]) -> str: + return hashlib.sha256(("\n".join(entries) + "\n").encode()).hexdigest() + + +def graph_contract(site: Path, expectations: ProjectionExpectations) -> None: + graph_path = site / "graph.json" + if not graph_path.is_file(): + raise BuildError("Static artifact has no graph.json") + graph = json.loads(graph_path.read_text(encoding="utf-8")) + nodes = graph.get("nodes", []) + edges = graph.get("edges", []) + if {node.get("id") for node in nodes} != expectations.graph_node_pids or len( + nodes + ) != len(expectations.graph_node_pids): + raise BuildError("Static graph nodes do not match the source inventory") + pairs = {(edge.get("source"), edge.get("target")) for edge in edges} + if pairs != expectations.graph_edges or len(edges) != len(expectations.graph_edges): + raise BuildError("Static graph edges do not match native relationships") + + +def entity_routes(site: Path, expected_routes: Sequence[str] = ()) -> set[str]: + routes: set[str] = set() + declared_sections = {route.split("/", 1)[0] for route in expected_routes if route} + for section in ENTITY_SECTIONS | declared_sections: + root = site / section + if not root.is_dir(): + continue + for path in root.rglob("index.html"): + relative = path.parent.relative_to(site).as_posix() + if relative != section: + routes.add(relative) + return routes + + +def published_html_routes(site: Path) -> set[str]: + """Return every non-home HTML route, including flat aliases.""" + routes: set[str] = set() + for path in site.rglob("*.html"): + if path.name == "index.html": + if path.parent != site: + routes.add(path.parent.relative_to(site).as_posix()) + else: + routes.add(path.relative_to(site).as_posix()) + return routes + + +def quicklink_references(site: Path) -> list[str]: + """Return built HTML/JavaScript files that retain Quicklink prefetching.""" + references: list[str] = [] + for path in sorted(site.rglob("*")): + if not path.is_file() or path.suffix not in {".html", ".js"}: + continue + if b"quicklink" in path.read_bytes().lower(): + references.append(path.relative_to(site).as_posix()) + return references + + +def declared_taxonomy_routes() -> frozenset[str]: + """Derive framework-owned list routes from the pinned upstream config.""" + taxonomies = tomllib.loads(TAXONOMY_CONFIG.read_text(encoding="utf-8")) + routes = list(taxonomies.values()) + if ( + not routes + or not all( + isinstance(route, str) + and re.fullmatch(r"[a-z0-9][a-z0-9-]*", route) is not None + for route in routes + ) + or len(routes) != len(set(routes)) + ): + raise BuildError("Upstream taxonomy routes are invalid or duplicated") + return frozenset(routes) + + +def verify_published_route_closure( + site: Path, + expected_entity_routes: Sequence[str], + expected_editorial_routes: Sequence[str], + expected_taxonomy_routes: Sequence[str] = (), + expected_alias_routes: Sequence[str] = (), + expected_framework_routes: Sequence[str] = (), +) -> None: + """Reject missing or undeclared generated/editorial HTML routes.""" + expected = ( + set(expected_entity_routes) + | set(expected_editorial_routes) + | set(expected_taxonomy_routes) + | set(expected_alias_routes) + | set(expected_framework_routes) + ) + actual = published_html_routes(site) + if actual != expected: + raise BuildError( + "Static HTML route closure disagrees with presentation/source data: " + f"missing={sorted(expected - actual)}, " + f"undeclared={sorted(actual - expected)}" + ) + + +def verify_site( + site: Path, + base_url: str, + asset_manifest: dict[str, Any], + asset_files: dict[str, Path], + expectations: ProjectionExpectations | None = None, + presentation: PresentationContract | None = None, +) -> dict[str, Any]: + contract = load_projection_contract() + records = source_closure(contract) + if expectations is None: + expectations = validate_record_contract(records, contract) + if presentation is None: + presentation = validate_presentation_contract(records, contract.homepage_pid) + taxonomy_routes = declared_taxonomy_routes() + required = { + "index.html", + "graph.js", + "graph.json", + "site.webmanifest", + "apple-touch-icon.png", + "favicon-16x16.png", + "favicon-32x32.png", + "explore/index.html", + "mstile-150x150.png", + *(f"{route}/index.html" for route in expectations.entity_routes), + *(f"{route}/index.html" for route in presentation.editorial_routes), + *(f"{route}/index.html" for route in taxonomy_routes), + *presentation.editorial_aliases, + "404.html", + } + missing = sorted(path for path in required if not (site / path).is_file()) + if missing: + raise BuildError(f"Static CON artifact is missing routes/assets: {missing}") + verify_published_route_closure( + site, + expectations.entity_routes, + presentation.editorial_routes, + taxonomy_routes, + presentation.editorial_aliases, + {"404.html"}, + ) + routes = entity_routes(site, expectations.entity_routes) + if routes != expectations.entity_routes: + raise BuildError( + "German or unexpected entity routes leaked into the CON artifact: " + f"{sorted(routes)}" + ) + graph_contract(site, expectations) + + for target, source in artifact_asset_targets(asset_manifest).items(): + output = site / target + declared = asset_files.get(source) + if declared is None or not output.is_file(): + raise BuildError(f"Declared site asset is absent: {target}") + if ( + hashlib.sha256(output.read_bytes()).digest() + != hashlib.sha256(declared.read_bytes()).digest() + ): + raise BuildError(f"Declared site asset is stale: {target}") + + homepage = (site / "index.html").read_text(encoding="utf-8") + if "Center for Open Neuroscience" not in homepage: + raise BuildError("Homepage branding does not identify CON") + if "con-logo.png" not in homepage: + raise BuildError("Homepage does not use the CON logo") + upstream_branding = { + "https://www.fz-juelich.de/", + "https://www.medizin.hhu.de/", + } + leaked_links = sorted(link for link in upstream_branding if link in homepage) + forbidden_assets = { + "android-chrome-192x192.png", + "android-chrome-512x512.png", + "img/fzj.svg", + "img/hhu.svg", + } + leaked_assets = sorted(name for name in forbidden_assets if (site / name).exists()) + if leaked_links or leaked_assets: + raise BuildError( + "Upstream institutional branding leaked into the CON artifact: " + f"links={leaked_links}, assets={leaked_assets}" + ) + quicklink_files = quicklink_references(site) + if quicklink_files: + raise BuildError( + f"Quicklink prefetching leaked into the CON artifact: {quicklink_files}" + ) + base_path = urlsplit(base_url).path or "/" + expected_explore = f"{base_path.rstrip('/')}/explore" + unquoted_homepage = homepage.replace('"', "").replace("'", "") + if f"href={expected_explore}" not in unquoted_homepage: + raise BuildError("Homepage Explore link does not target the local static route") + header_logo = role_asset( + asset_manifest, + asset_files, + "upstream-compatible-header-brand", + ) + expected_logo = hashlib.sha256(header_logo.read_bytes()).hexdigest() + if not any( + path.is_file() + and hashlib.sha256(path.read_bytes()).hexdigest() == expected_logo + for path in site.rglob("*") + ): + raise BuildError("The committed CON logo is absent from the site") + manifest = json.loads((site / "site.webmanifest").read_text(encoding="utf-8")) + if manifest.get("name") != "Center for Open Neuroscience": + raise BuildError("The static web manifest does not identify CON") + if "Congo" in json.dumps(manifest): + raise BuildError("Upstream Congo branding leaked into the web manifest") + for name in ( + "apple-touch-icon.png", + "favicon-16x16.png", + "favicon-32x32.png", + "mstile-150x150.png", + ): + if hashlib.sha256((site / name).read_bytes()).hexdigest() != expected_logo: + raise BuildError(f"Static CON branding is stale: {name}") + person = site / "persons" / "yaroslav-halchenko" / "index.html" + person_text = person.read_text(encoding="utf-8") + if "Yaroslav" not in person_text: + raise BuildError("Person page does not identify Yaroslav") + + audit = run( + [ + sys.executable, + ROOT / "tools" / "adapt_upstream_pages.py", + site, + "--base-path", + base_path, + "--edit-url", + os.environ.get("SHACL_VUE_URL", DEFAULT_EDIT_URL), + "--check-only", + ], + action="Audit static CON base-path links", + ) + entries = manifest_entries(site) + return { + "base_url": base_url, + "entity_routes": sorted(routes), + "editorial_routes": sorted(presentation.editorial_routes), + "editorial_aliases": sorted(presentation.editorial_aliases), + "taxonomy_routes": sorted(taxonomy_routes), + "files": len(entries), + "manifest_sha256": manifest_digest(entries), + "path_audit": audit.strip(), + } + + +def build_site(destination: Path, base_url: str) -> dict[str, Any]: + destination = safe_destination(destination) + try: + profile = load_yaml(PROFILE_ROOT / "profile.yaml") + verify_final_site_state(profile) + verify_declared_pins(profile) + contract = load_projection_contract(profile) + source_records = source_closure(contract) + expectations = validate_record_contract(source_records, contract) + presentation = validate_presentation_contract( + source_records, contract.homepage_pid + ) + verify_manifest(COMMITTED) + verify_assembly_manifest() + records = [ + json.loads(line) + for line in (COMMITTED / "records.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ] + validate_projection(records, COMMITTED, expectations) + stack_records(records, BUILD_ROOT / "records.jsonl") + asset_manifest = load_asset_manifest(ASSET_MANIFEST) + asset_files = hydrate_all_assets() + except (ProjectionError, AssetError) as error: + raise BuildError(str(error)) from error + try: + assemble_source(asset_manifest, asset_files) + except AssetError as error: + raise BuildError(str(error)) from error + if destination.exists(): + shutil.rmtree(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + environment = os.environ.copy() + environment["HUGO_ENVIRONMENT"] = "con" + version = run(["hugo", "version"], action="Inspect Hugo version") + if "hugo v0.154.5" not in version or "extended" not in version: + raise BuildError(f"Unexpected Hugo runtime: {version.strip()}") + run( + [ + "hugo", + "--minify", + "--cleanDestinationDir", + "--environment", + "con", + "--source", + ASSEMBLY, + "--destination", + destination, + "--baseURL", + base_url, + ], + environment=environment, + action="Build the backend-free CON site", + ) + base_path = urlsplit(base_url).path or "/" + run( + [ + sys.executable, + ROOT / "tools" / "adapt_upstream_pages.py", + destination, + "--base-path", + base_path, + "--edit-url", + os.environ.get("SHACL_VUE_URL", DEFAULT_EDIT_URL), + ], + action="Adapt generated CON paths and edit links", + ) + report = verify_site( + destination, + base_url, + asset_manifest, + asset_files, + expectations, + presentation, + ) + report_path = destination.parent / f"{destination.name}-build.json" + report_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + manifest_path = destination.parent / f"{destination.name}-manifest.sha256" + manifest_path.write_text( + "\n".join(manifest_entries(destination)) + "\n", + encoding="utf-8", + ) + return report + + +def compare_builds( + first: Path, + second: Path, + base_url: str, +) -> dict[str, Any]: + first_report = build_site(first, base_url) + second_report = build_site(second, base_url) + first_entries = manifest_entries(first) + second_entries = manifest_entries(second) + if first_entries != second_entries: + raise BuildError("Two clean static builds are not byte-identical") + return { + "first": first_report, + "second": second_report, + "byte_identical": True, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--destination", + type=Path, + default=Path(os.environ.get("DESTINATION", DEFAULT_DESTINATION)), + ) + action = parser.add_mutually_exclusive_group() + action.add_argument( + "--update-assembly-manifest", + action="store_true", + help="replace the reviewed static-assembly input digest", + ) + action.add_argument( + "--check-assembly-manifest", + action="store_true", + help="verify only the reviewed static-assembly input digest", + ) + parser.add_argument( + "--base-url", + default=os.environ.get("BASE_URL", DEFAULT_BASE_URL), + ) + parser.add_argument( + "--repeat-destination", + type=Path, + help="also build here and require byte-identical output", + ) + args = parser.parse_args() + base_url = args.base_url.rstrip("/") + "/" + try: + if args.update_assembly_manifest: + print(f"Updated {update_assembly_manifest()}") + return 0 + if args.check_assembly_manifest: + verify_assembly_manifest() + print(f"Verified {assembly_manifest_path()}") + return 0 + if args.repeat_destination: + report = compare_builds( + args.destination, + args.repeat_destination, + base_url, + ) + else: + report = build_site(args.destination, base_url) + print(json.dumps(report, sort_keys=True)) + except BuildError as error: + print(f"clean-migration build: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_pages_editor.py b/tools/build_pages_editor.py new file mode 100644 index 0000000..3e9032b --- /dev/null +++ b/tools/build_pages_editor.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Build the credential-free SHACL Vue editor for the CON Pages preview.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +from typing import Any, Sequence + +from dump_things_service import Format +from dump_things_service.converter import FormatConverter +from rdflib import Graph +from rdflib.compare import to_canonical_graph + +from build_con_site import ( + BuildError, + ROOT, + manifest_digest, + manifest_entries, + safe_destination, +) +from con_projection import ( + SCHEMA, + SITE, + git_commit, + load_projection_contract, + source_closure, + validate_record_contract, +) + + +UI = ROOT / "submodules" / "pool.psychoinformatics.de-ui" +DEFAULT_SOURCE = UI / "dist" / "ui" +DEFAULT_DESTINATION = ROOT / "build" / "pages-editor" +DEFAULT_REPEAT_DESTINATION = ROOT / "build" / "pages-editor-repeat" +TEXT_CONFIG = { + "app_name": "CON metadata review", + "app_theme": { + "active_color": "#2b71b9", + "hover_color": "#2b71b9", + "link_color": "#7fa7d8", + "logo": "logo.png", + "panel_color": "#29343e", + "visited_color": "#7fa7d8", + }, + "class_url": "dlschemas_owl.ttl", + "data_url": "records.ttl", + "documentation_url": "", + "external_config_url": "config_default_xyzri.yaml", + "front_page_content": ( + "Edit a public CON record, save it in the form, then use the download " + "button to create a review bundle. The browser has no write service " + "or authentication credential." + ), + "page_title": "CON metadata review", + "priority_classes": [ + { + "class": "dlthings:Thing", + "icon": "mdi-view-list", + "include_subclasses": True, + "title": "All", + } + ], + "review_bundle_catalog": "record-sources.json", + "review_bundle_mode": "patch-download", + "shapes_url": "dlschemas_shacl.ttl", + "source_code_url": "https://github.com/con/orinoco-lite-dev", + "use_default_classes": False, + "use_default_data": False, + "use_default_shapes": False, + "use_service": False, + "use_token": False, +} + + +def write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def copy_ui(source: Path, destination: Path) -> None: + if not source.is_dir() or not (source / "index.html").is_file(): + raise BuildError("Built pool UI is missing index.html") + for candidate in source.rglob("*"): + if candidate.is_symlink(): + raise BuildError( + f"Built pool UI contains a symlink: {candidate.relative_to(source)}" + ) + shutil.copytree(source, destination) + for name in ( + "config.json", + "config.yaml", + "config.yml", + "dlschemas_data.ttl", + ): + destination.joinpath(name).unlink(missing_ok=True) + for source_map in destination.rglob("*.map"): + source_map.unlink() + + +def static_records_turtle() -> tuple[str, int]: + contract = load_projection_contract() + records = source_closure(contract) + validate_record_contract(records, contract) + converter = FormatConverter(str(SCHEMA), Format.json, Format.ttl) + rendered: list[str] = [] + for source in sorted(records, key=lambda item: item.record["pid"]): + try: + turtle = converter.convert(source.record, source.class_name) + except Exception as error: + raise BuildError( + f"Could not render editor RDF for {source.record['pid']}: {error}" + ) from error + rendered.append(turtle) + return canonical_turtle(rendered), len(records) + + +def canonical_turtle(snippets: Sequence[str]) -> str: + """Return deterministic RDF accepted by a Turtle parser. + + RDFLib's Turtle serializer can emit equivalent blank-node properties and + repeated values in process-dependent order. Canonical blank-node labels + plus sorted N-Triples make the byte stream stable. N-Triples is a strict + subset of Turtle, so SHACL Vue can continue to load ``records.ttl``. + """ + + graph = Graph() + for snippet in snippets: + graph.parse(data=snippet, format="turtle") + serialized = to_canonical_graph(graph).serialize(format="nt") + lines = sorted(line for line in serialized.splitlines() if line.strip()) + return "\n".join(lines) + "\n" + + +def tree_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted( + candidate for candidate in root.rglob("*") if candidate.is_file() + ): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(relative + b"\0" + hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def build_editor(destination: Path, source: Path = DEFAULT_SOURCE) -> dict[str, Any]: + destination = safe_destination(destination) + if destination.exists(): + shutil.rmtree(destination) + copy_ui(source.resolve(), destination) + turtle, record_count = static_records_turtle() + destination.joinpath("records.ttl").write_text(turtle, encoding="utf-8") + write_json(destination / "config.json", TEXT_CONFIG) + contract = { + "authentication": "none", + "backend": "none", + "input_sha256": tree_digest(destination), + "mode": "patch-download", + "pool_ui_commit": git_commit(UI), + "record_count": record_count, + "schema_commit": git_commit(ROOT / "submodules" / "things-schemas"), + "site_commit": git_commit(SITE), + "version": 1, + } + write_json(destination / "editor-contract.json", contract) + return contract + + +def build_pool_ui() -> None: + """Build the pinned UI without reusing a previous distribution tree.""" + + result = subprocess.run( + ["make", "-C", str(UI), "build-ui"], + cwd=ROOT, + check=False, + ) + if result.returncode: + raise BuildError(f"Pinned pool UI build failed ({result.returncode})") + + +def verify_editor_builds( + destination: Path, + repeat_destination: Path, + source: Path = DEFAULT_SOURCE, +) -> dict[str, Any]: + """Build the UI/editor twice independently and compare exact bytes.""" + + if source.resolve() != DEFAULT_SOURCE.resolve(): + raise BuildError("Repeated editor verification requires the pinned UI source") + build_pool_ui() + first = build_editor(destination, source) + first_entries = manifest_entries(destination) + build_pool_ui() + second = build_editor(repeat_destination, source) + second_entries = manifest_entries(repeat_destination) + if first_entries != second_entries: + raise BuildError("Two independent static editor builds are not byte-identical") + return { + "byte_identical": True, + "files": len(first_entries), + "first": first, + "manifest_sha256": manifest_digest(first_entries), + "second": second, + } + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--destination", type=Path, default=DEFAULT_DESTINATION) + parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) + parser.add_argument("--repeat-destination", type=Path) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.repeat_destination is None: + report = build_editor(args.destination, args.source) + else: + report = verify_editor_builds( + args.destination, + args.repeat_destination, + args.source, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_upstream_site.sh b/tools/build_upstream_site.sh new file mode 100755 index 0000000..2b39bd9 --- /dev/null +++ b/tools/build_upstream_site.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +site_root="$repository_root/submodules/www-from-model" +base_url=${BASE_URL:-http://127.0.0.1:1313/} +destination=${DESTINATION:-$repository_root/build/upstream-psychoinformatics} +edit_url=${SHACL_VUE_URL:-https://pool.psychoinformatics.de/ui/} +annex_commit=010ca44f751d2ab60b9d4ad58c5931d1804e3c9e +upstream_url=https://hub.psychoinformatics.de/www/www-from-model.git +annex_remote_name=clean-migration-upstream-$$ +annex_remote_ref=refs/remotes/$annex_remote_name/git-annex + +original_core_worktree= +had_core_worktree=false +if original_core_worktree=$(git -C "$site_root" config --get core.worktree 2>/dev/null); then + had_core_worktree=true +fi + +restore_local_state() { + git -C "$site_root" update-ref -d "$annex_remote_ref" 2>/dev/null || true + git -C "$site_root" config --remove-section \ + "remote.$annex_remote_name" 2>/dev/null || true + if [[ "$had_core_worktree" == true ]]; then + git -C "$site_root" config core.worktree "$original_core_worktree" + else + git -C "$site_root" config --unset-all core.worktree 2>/dev/null || true + fi +} +trap restore_local_state EXIT + +site_git() { + git -c core.worktree="$site_root" -C "$site_root" "$@" +} + +base_url=${base_url%/}/ +destination=$(python3 -c \ + 'import sys; from pathlib import Path; print(Path(sys.argv[1]).resolve())' \ + "$destination") +case "$destination/" in + "$repository_root/build/"* | /tmp/* | /private/tmp/*) ;; + *) + echo "DESTINATION must be below $repository_root/build or a temporary directory" >&2 + exit 2 + ;; +esac +base_path=$(python3 -c \ + 'import sys; from urllib.parse import urlsplit; print(urlsplit(sys.argv[1]).path or "/")' \ + "$base_url") + +git -C "$repository_root" submodule sync -- submodules/www-from-model +if git -C "$site_root" rev-parse --git-dir >/dev/null 2>&1; then + if [[ -n "$(site_git status --porcelain)" ]]; then + echo "Refusing to update a modified www-from-model worktree" >&2 + exit 2 + fi +fi +git -C "$repository_root" submodule update --init --depth 1 -- submodules/www-from-model +site_git submodule update --init --depth 1 -- themes/congo + +site_git fetch --no-write-fetch-head \ + "$upstream_url" "+$annex_commit:$annex_remote_ref" +site_git -c annex.private=true annex init +site_git \ + -c annex.private=true \ + -c remote.$annex_remote_name.url="$upstream_url" \ + -c remote.$annex_remote_name.fetch=+refs/heads/\*:refs/remotes/$annex_remote_name/\* \ + annex get --from "$annex_remote_name" . +test -z "$(site_git -c annex.private=true annex find --not --in=here)" + +hugo version | grep -q 'hugo v0\.154\.5.*extended' +hugo \ + --minify \ + --cleanDestinationDir \ + --source "$site_root" \ + --destination "$destination" \ + --baseURL "$base_url" + +python3 "$repository_root/tools/adapt_upstream_pages.py" \ + "$destination" \ + --base-path "$base_path" \ + --edit-url "$edit_url" +python3 "$repository_root/tools/adapt_upstream_pages.py" \ + "$destination" \ + --base-path "$base_path" \ + --edit-url "$edit_url" \ + --check-only + +printf 'Built the upstream site at %s\n' "$destination" diff --git a/tools/check_local_stack.py b/tools/check_local_stack.py new file mode 100644 index 0000000..5897a10 --- /dev/null +++ b/tools/check_local_stack.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +"""Check the four-collection clean-migration local stack contract.""" + +from __future__ import annotations + +import html +import hashlib +import json +import os +import re +import sys +import uuid +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import SplitResult, parse_qs, urlencode, urlsplit +from urllib.request import Request, urlopen + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SITE = Path( + os.environ.get( + "CON_SITE_ROOT", + ROOT / "submodules" / "centerforopenneuroscience.org", + ) +).resolve() +STACK = ROOT / "build" / "local-stack" +UPSTREAM_SNAPSHOT = STACK / "pool" / "public-thing.jsonl" +CON_RECORDS = ROOT / "build" / "con-projection" / "records.jsonl" +CON_SITE = ROOT / "build" / "con-site" +EDITOR_TOKEN = STACK / "editor-token" +SEED_TOKEN = STACK / "seed-token" +SERVICE_URL = "http://127.0.0.1:8111" +EDITOR_URL = "http://127.0.0.1:3000/" +COLLECTIONS = { + "upstream-public", + "upstream-protected", + "con-public", + "con-protected", +} +LEGACY_COLLECTIONS = ("public", "protected") +EDIT_LINK = re.compile(r"href=(?:\"(?P[^\"]+)\"|'(?P[^']+)')") +PROBE_CLASS = "XYZProject" +LEGACY_PROBE_PID = "xyzrins:projects/_clean-migration-write-probe" +PROBE_PID_PREFIX = f"{LEGACY_PROBE_PID}-" +CON_PERSON_PID = "xyzrins:persons/yaroslav-halchenko" +REPRESENTATIVE_EDIT_PIDS = frozenset( + { + CON_PERSON_PID, + "xyzrins:projects/datalad", + } +) +PROJECTION_CONTRACT = SITE / "profiles" / "con" / "projection.yaml" +EDIT_QUERY_KEYS = frozenset({"sh:NodeShape", "pid", "edit"}) + + +def request_json( + method: str, + url: str, + token: str | None, + body: object | None = None, + *, + missing_ok: bool = False, +) -> object | None: + headers = {"Accept": "application/json"} + if token is not None: + headers["X-DumpThings-Token"] = token + data = None + if body is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(body).encode("utf-8") + request = Request(url, headers=headers, data=data, method=method) + try: + with urlopen(request, timeout=30) as response: + raw = response.read() + except HTTPError as error: + if missing_ok and error.code == 404: + return None + detail = error.read().decode("utf-8", errors="replace") + raise RuntimeError( + f"{method} {url} failed ({error.code}): {detail[:500]}" + ) from error + except URLError as error: + raise RuntimeError(f"Could not reach {url}: {error}") from error + return json.loads(raw) if raw else None + + +def expect_rejected( + method: str, + url: str, + token: str | None, + body: object, +) -> None: + """Require an API request to fail for lack of write permission.""" + headers = {"Accept": "application/json", "Content-Type": "application/json"} + if token is not None: + headers["X-DumpThings-Token"] = token + request = Request( + url, + headers=headers, + data=json.dumps(body).encode("utf-8"), + method=method, + ) + try: + with urlopen(request, timeout=30) as response: + response.read() + except HTTPError as error: + error.read() + if error.code in {401, 403}: + return + raise RuntimeError( + f"Unauthorized probe returned unexpected status {error.code}: {url}" + ) from error + except URLError as error: + raise RuntimeError(f"Could not reach {url}: {error}") from error + raise RuntimeError(f"Unauthorized write unexpectedly succeeded: {url}") + + +def read_text(url: str) -> str: + with urlopen(url, timeout=30) as response: + return response.read().decode("utf-8") + + +def normalize_record(record: dict) -> dict: + """Match the service's omission of a top-level class discriminator.""" + normalized = dict(record) + normalized.pop("schema_type", None) + return normalized + + +def manifest_records(path: Path) -> dict[str, dict]: + records: dict[str, dict] = {} + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, start=1): + try: + item = json.loads(line) + record = item["record"] + pid = record["pid"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise RuntimeError( + f"{path}:{line_number}: invalid stack JSONL envelope" + ) from error + if not isinstance(record, dict): + raise RuntimeError(f"{path}:{line_number}: record must be an object") + if not isinstance(pid, str) or not pid: + raise RuntimeError(f"{path}:{line_number}: record pid must be a string") + if pid in records: + raise RuntimeError( + f"{path}:{line_number}: duplicate record pid {pid!r}" + ) + records[pid] = normalize_record(record) + if not records: + raise RuntimeError(f"{path}: manifest has no records") + return records + + +def manifest_envelopes(path: Path) -> dict[str, dict]: + """Load the generated service envelopes, retaining their class names.""" + envelopes: dict[str, dict] = {} + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, start=1): + try: + item = json.loads(line) + class_name = item["class_name"] + record = item["record"] + pid = record["pid"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise RuntimeError( + f"{path}:{line_number}: invalid stack JSONL envelope" + ) from error + if not isinstance(class_name, str) or not class_name: + raise RuntimeError(f"{path}:{line_number}: class_name must be a string") + if not isinstance(record, dict) or not isinstance(pid, str) or not pid: + raise RuntimeError( + f"{path}:{line_number}: record and pid must be valid" + ) + schema_type = record.get("schema_type") + if schema_type != f"xyzri:{class_name}": + raise RuntimeError( + f"{path}:{line_number}: class_name {class_name!r} does not " + f"match schema_type {schema_type!r}" + ) + if pid in envelopes: + raise RuntimeError( + f"{path}:{line_number}: duplicate record pid {pid!r}" + ) + envelopes[pid] = item + if not envelopes: + raise RuntimeError(f"{path}: manifest has no records") + return envelopes + + +def expected_edit_pids( + records_path: Path = CON_RECORDS, + projection_path: Path = PROJECTION_CONTRACT, +) -> frozenset[str]: + """Derive the editable route closure from records and render policy.""" + try: + contract = yaml.safe_load(projection_path.read_text(encoding="utf-8")) + render = contract["render"] + pages = render["pages"] + homepage_pid = render["homepage"]["pid"] + except (OSError, KeyError, TypeError, yaml.YAMLError) as error: + raise RuntimeError( + f"Invalid CON projection contract: {projection_path}" + ) from error + if not isinstance(pages, dict) or not pages: + raise RuntimeError("CON projection contract declares no rendered classes") + rendered_types = set(pages) + if not all( + isinstance(schema_type, str) and schema_type.startswith("xyzri:") + for schema_type in rendered_types + ): + raise RuntimeError("CON rendered class designators must be xyzri CURIEs") + if not isinstance(homepage_pid, str) or not homepage_pid: + raise RuntimeError("CON projection homepage pid is invalid") + + envelopes = manifest_envelopes(records_path) + expected = frozenset( + pid + for pid, item in envelopes.items() + if item["record"]["schema_type"] in rendered_types + ) + if homepage_pid not in expected: + raise RuntimeError(f"CON homepage {homepage_pid!r} is not a rendered record") + missing_smoke = REPRESENTATIVE_EDIT_PIDS - expected + if missing_smoke: + raise RuntimeError( + "Representative CON edit fixtures are not rendered: " + f"{sorted(missing_smoke)!r}" + ) + return expected + + +def curated_records(collection: str, token: str) -> dict[str, dict]: + records: dict[str, dict] = {} + page = 1 + while True: + query = urlencode({"page": page, "size": 100}) + url = f"{SERVICE_URL}/{collection}/curated/records/p/?{query}" + payload = request_json("GET", url, token) + if not isinstance(payload, dict) or not isinstance(payload.get("items"), list): + raise RuntimeError(f"Unexpected paginated response from {url}") + for record in payload["items"]: + pid = record.get("pid") if isinstance(record, dict) else None + if not isinstance(pid, str): + raise RuntimeError(f"Record without a pid in {collection}") + if pid in records: + raise RuntimeError(f"Duplicate record {pid!r} in {collection}") + records[pid] = normalize_record(record) + pages = int(payload.get("pages", 1)) + if page >= pages: + return records + page += 1 + + +def record_digest(record: dict) -> str: + payload = json.dumps(record, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def describe_difference( + expected: dict[str, dict], + actual: dict[str, dict], +) -> str: + missing = sorted(expected.keys() - actual.keys())[:3] + extra = sorted(actual.keys() - expected.keys())[:3] + changed = [ + ( + pid, + record_digest(expected[pid])[:12], + record_digest(actual[pid])[:12], + ) + for pid in sorted(expected.keys() & actual.keys()) + if expected[pid] != actual[pid] + ][:3] + return f"missing={missing!r}, extra={extra!r}, changed={changed!r}" + + +def check_seed_separation(token: str) -> dict[str, int]: + upstream = manifest_records(UPSTREAM_SNAPSHOT) + con = manifest_records(CON_RECORDS) + expected = { + "upstream-public": upstream, + "upstream-protected": upstream, + "con-public": con, + "con-protected": con, + } + counts: dict[str, int] = {} + for collection, expected_records in expected.items(): + actual = curated_records(collection, token) + if actual != expected_records: + difference = describe_difference(expected_records, actual) + raise RuntimeError(f"Curated records differ in {collection}: {difference}") + counts[collection] = len(actual) + return counts + + +def check_no_legacy_collection_stores() -> None: + legacy = [ + STACK / "store" / collection + for collection in LEGACY_COLLECTIONS + if (STACK / "store" / collection).exists() + ] + if legacy: + raise RuntimeError(f"Obsolete local collection stores remain: {legacy}") + + +def check_editor_ui() -> None: + config = read_text(f"{EDITOR_URL}config.yaml") + external = read_text(f"{EDITOR_URL}config_default_xyzri.yaml") + service_url = f"{SERVICE_URL}/con-protected/" + if config.count(service_url) != 2: + raise RuntimeError( + "SHACL Vue must use con-protected for both read and write URLs" + ) + for forbidden in ( + f"{SERVICE_URL}/con-public/", + f"{SERVICE_URL}/upstream-public/", + f"{SERVICE_URL}/upstream-protected/", + "https://pool.psychoinformatics.de/api/", + ): + if forbidden in config: + raise RuntimeError(f"SHACL Vue unexpectedly references {forbidden!r}") + for required in ( + "use_service: true", + "use_token: true", + "http://127.0.0.1:8122/git-annex", + ): + if required not in config: + raise RuntimeError(f"SHACL Vue configuration is missing {required!r}") + for required in ("xyzrins:", "dlschemas_owl.ttl", "data_url: ''"): + if required not in external: + raise RuntimeError( + f"SHACL Vue external configuration is missing {required!r}" + ) + + +def check_static_edit_links( + expected_pids: frozenset[str] | None = None, +) -> int: + if expected_pids is None: + expected_pids = expected_edit_pids() + links: list[tuple[str, SplitResult, dict[str, list[str]]]] = [] + for path in sorted(CON_SITE.rglob("*.html")): + source = path.read_text(encoding="utf-8") + for match in EDIT_LINK.finditer(source): + link = html.unescape(match.group("double") or match.group("single")) + parsed = urlsplit(link) + query = parse_qs(parsed.query, keep_blank_values=True) + if ( + parsed.hostname == "127.0.0.1" + and parsed.port == 3000 + or "edit" in query + or "sh:NodeShape" in query + ): + links.append((link, parsed, query)) + if not links: + raise RuntimeError(f"No edit links found in generated CON site: {CON_SITE}") + linked_pids: list[str] = [] + for link, parsed, query in links: + if ( + parsed.scheme != "http" + or parsed.netloc != "127.0.0.1:3000" + or parsed.path != "/" + or parsed.fragment + or set(query) != EDIT_QUERY_KEYS + or query.get("sh:NodeShape") != ["dlthings:Thing"] + or len(query.get("pid", [])) != 1 + or query.get("edit") != ["true"] + ): + raise RuntimeError( + "CON static edit link does not use the credential-free local " + f"CON editor contract: {link}" + ) + linked_pids.append(query["pid"][0]) + actual_pids = set(linked_pids) + if actual_pids != expected_pids or len(linked_pids) != len(expected_pids): + raise RuntimeError( + "CON static edit links do not match the rendered record set: " + f"expected={sorted(expected_pids)!r}, " + f"actual={sorted(linked_pids)!r}" + ) + return len(links) + + +def incoming_record( + collection: str, + token: str, + pid: str, +) -> object | None: + query = urlencode({"pid": pid}) + url = f"{SERVICE_URL}/{collection}/incoming/local-editor/record?{query}" + return request_json("GET", url, token, missing_ok=True) + + +def curated_record( + collection: str, + token: str | None, + pid: str, +) -> object | None: + query = urlencode({"pid": pid}) + url = f"{SERVICE_URL}/{collection}/curated/record?{query}" + return request_json("GET", url, token, missing_ok=True) + + +def check_anonymous_con_read() -> None: + """Require the editor's default identity to read curated CON records.""" + query = urlencode({"pid": CON_PERSON_PID, "format": "json"}) + url = f"{SERVICE_URL}/con-protected/record?{query}" + record = request_json("GET", url, None) + if not isinstance(record, dict) or record.get("pid") != CON_PERSON_PID: + raise RuntimeError( + "Anonymous con-protected read did not return the curated " + f"CON person {CON_PERSON_PID!r}" + ) + + +def delete_incoming_record(collection: str, token: str, pid: str) -> None: + query = urlencode({"pid": pid}) + url = f"{SERVICE_URL}/{collection}/incoming/local-editor/record?{query}" + request_json("DELETE", url, token, missing_ok=True) + + +def incoming_probe_pids(collection: str, token: str) -> set[str]: + """Find every reserved probe left by an interrupted acceptance run.""" + found: set[str] = set() + page = 1 + while True: + query = urlencode({"page": page, "size": 100}) + url = f"{SERVICE_URL}/{collection}/incoming/local-editor/records/p/?{query}" + payload = request_json("GET", url, token, missing_ok=True) + if payload is None: + return found + if not isinstance(payload, dict) or not isinstance(payload.get("items"), list): + raise RuntimeError(f"Unexpected paginated incoming response from {url}") + for record in payload["items"]: + pid = record.get("pid") if isinstance(record, dict) else None + if not isinstance(pid, str): + raise RuntimeError(f"Incoming record without a pid in {collection}") + if pid == LEGACY_PROBE_PID or pid.startswith(PROBE_PID_PREFIX): + found.add(pid) + pages = int(payload.get("pages", 1)) + if page >= pages: + return found + page += 1 + + +def prove_write_isolation(editor_token: str, seed_token: str) -> None: + canonical_pids = set(manifest_envelopes(CON_RECORDS)) + collisions = sorted( + pid + for pid in canonical_pids + if pid == LEGACY_PROBE_PID or pid.startswith(PROBE_PID_PREFIX) + ) + if collisions: + raise RuntimeError( + f"Canonical records use the reserved acceptance PID namespace: {collisions}" + ) + probe_pid = f"{PROBE_PID_PREFIX}{uuid.uuid4().hex}" + probe = {"pid": probe_pid, "schema_type": "xyzri:XYZProject"} + url = f"{SERVICE_URL}/con-protected/record/{PROBE_CLASS}" + cleanup_pids = {LEGACY_PROBE_PID, probe_pid} + for collection in COLLECTIONS: + cleanup_pids.update(incoming_probe_pids(collection, seed_token)) + try: + for stale_pid in sorted(cleanup_pids): + for collection in COLLECTIONS: + delete_incoming_record(collection, seed_token, stale_pid) + for collection in sorted(COLLECTIONS): + boundary_url = f"{SERVICE_URL}/{collection}/record/{PROBE_CLASS}" + expect_rejected("POST", boundary_url, None, probe) + for collection in sorted(COLLECTIONS - {"con-protected"}): + boundary_url = f"{SERVICE_URL}/{collection}/record/{PROBE_CLASS}" + expect_rejected("POST", boundary_url, editor_token, probe) + + request_json("POST", url, editor_token, probe) + incoming = { + collection: incoming_record(collection, seed_token, probe_pid) + for collection in COLLECTIONS + } + protected_record = incoming["con-protected"] + if ( + not isinstance(protected_record, dict) + or protected_record.get("pid") != probe_pid + ): + raise RuntimeError( + "Editor write did not land in con-protected/incoming/local-editor" + ) + leaked = { + collection: record + for collection, record in incoming.items() + if collection != "con-protected" and record is not None + } + if leaked: + raise RuntimeError(f"Editor write leaked into incoming areas: {leaked}") + curated = { + collection: curated_record(collection, seed_token, probe_pid) + for collection in COLLECTIONS + } + if any(record is not None for record in curated.values()): + raise RuntimeError(f"Editor write leaked into curated areas: {curated}") + finally: + for stale_pid in sorted(cleanup_pids): + for collection in COLLECTIONS: + delete_incoming_record(collection, seed_token, stale_pid) + + +def main() -> int: + required = ( + EDITOR_TOKEN, + SEED_TOKEN, + UPSTREAM_SNAPSHOT, + CON_RECORDS, + CON_SITE, + ) + missing = [path for path in required if not path.exists()] + if missing: + raise RuntimeError(f"Missing local-stack inputs: {missing}") + editor_token = EDITOR_TOKEN.read_text(encoding="utf-8").strip() + seed_token = SEED_TOKEN.read_text(encoding="utf-8").strip() + server = request_json("GET", f"{SERVICE_URL}/server", seed_token) + if not isinstance(server, dict): + raise RuntimeError("Unexpected local Dump Things server response") + names = {item["name"] for item in server["collections"]} + if names != COLLECTIONS: + raise RuntimeError(f"Unexpected local collections: {sorted(names)}") + check_no_legacy_collection_stores() + counts = check_seed_separation(seed_token) + check_anonymous_con_read() + check_editor_ui() + edit_links = check_static_edit_links() + prove_write_isolation(editor_token, seed_token) + print( + "Local clean-migration stack healthy: " + f"{counts['upstream-public']} upstream records, " + f"{counts['con-public']} CON records, {edit_links} CON edit links" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: + print(f"Local stack check failed: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/tools/checkout_submodules.py b/tools/checkout_submodules.py new file mode 100644 index 0000000..755821f --- /dev/null +++ b/tools/checkout_submodules.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Fully check out every pinned development submodule. + +The static website build deliberately uses targeted shallow checkouts. This +helper is for development checkouts, where every top-level and nested +submodule must have complete history and match its parent's recorded gitlink. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +import subprocess +import sys +from typing import Iterable + + +class CheckoutError(RuntimeError): + """Report a submodule checkout or verification failure.""" + + +@dataclass(frozen=True) +class Gitlink: + """One gitlink recorded by a parent repository.""" + + parent: Path + path: Path + commit: str + display_path: Path + + @property + def checkout(self) -> Path: + return self.parent / self.path + + +def git( + repository: Path, + *arguments: str, + action: str | None = None, +) -> str: + """Run Git in ``repository`` and return stripped standard output.""" + command = ["git", "-C", str(repository), *arguments] + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout).strip() + description = action or "Git command failed" + if detail: + raise CheckoutError(f"{description}: {detail}") from error + raise CheckoutError( + f"{description}: command exited with status {error.returncode}" + ) from error + return result.stdout.strip() + + +def repository_root(path: Path) -> Path: + """Resolve ``path`` to the root of its containing Git worktree.""" + output = git( + path.resolve(), + "rev-parse", + "--show-toplevel", + action=f"Not a Git worktree: {path}", + ) + return Path(output).resolve() + + +def recorded_gitlinks(repository: Path) -> list[tuple[Path, str]]: + """Return all gitlink paths and commits recorded at ``HEAD``.""" + output = git( + repository, + "ls-tree", + "-r", + "-z", + "HEAD", + action=f"Unable to inspect gitlinks in {repository}", + ) + links: list[tuple[Path, str]] = [] + for entry in output.split("\0"): + if not entry: + continue + metadata, separator, name = entry.partition("\t") + fields = metadata.split() + if not separator or len(fields) != 3: + raise CheckoutError( + f"Unable to parse gitlink inventory entry in {repository}: " + f"{entry!r}" + ) + mode, object_type, commit = fields + if mode == "160000": + if object_type != "commit": + raise CheckoutError( + f"Gitlink {name!r} in {repository} has unexpected " + f"object type {object_type!r}" + ) + links.append((Path(name), commit)) + return links + + +def is_initialized(checkout: Path) -> bool: + """Return whether ``checkout`` is an initialized Git worktree.""" + if not (checkout / ".git").exists(): + return False + result = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return False + return Path(result.stdout.strip()).resolve() == checkout.resolve() + + +def is_shallow(checkout: Path) -> bool: + """Return whether an initialized submodule has shallow history.""" + output = git( + checkout, + "rev-parse", + "--is-shallow-repository", + action=f"Unable to inspect submodule history at {checkout}", + ) + if output not in {"true", "false"}: + raise CheckoutError( + f"Unexpected shallow-repository result at {checkout}: {output!r}" + ) + return output == "true" + + +def walk_initialized( + repository: Path, + prefix: Path = Path(), + visited: set[Path] | None = None, +) -> Iterable[Gitlink]: + """Yield initialized gitlinks recursively from the current checkouts.""" + if visited is None: + visited = set() + resolved = repository.resolve() + if resolved in visited: + raise CheckoutError(f"Recursive submodule cycle reaches {repository}") + visited.add(resolved) + + for path, commit in recorded_gitlinks(repository): + link = Gitlink(repository, path, commit, prefix / path) + if not is_initialized(link.checkout): + continue + yield link + yield from walk_initialized( + link.checkout, + link.display_path, + visited, + ) + + +def unshallow_initialized(repository: Path) -> int: + """Unshallow every currently initialized recursive submodule.""" + count = 0 + for link in list(walk_initialized(repository)): + if not is_shallow(link.checkout): + continue + print(f"Unshallowing {link.display_path}") + git( + link.checkout, + "fetch", + "--unshallow", + action=f"Unable to unshallow {link.display_path}", + ) + count += 1 + return count + + +def synchronize_submodules(repository: Path) -> None: + """Copy recursive URLs from each checked-out ``.gitmodules`` file.""" + git( + repository, + "submodule", + "sync", + "--recursive", + action="Unable to synchronize recursive submodule URLs", + ) + + +def update_submodules(repository: Path) -> None: + """Initialize and detach every recursive submodule at its gitlink.""" + git( + repository, + "submodule", + "update", + "--init", + "--recursive", + "--checkout", + "--no-recommend-shallow", + action="Unable to check out every recorded recursive gitlink", + ) + + +def verify_submodules(repository: Path) -> int: + """Verify initialization, full history, and exact recursive gitlinks.""" + errors: list[str] = [] + count = 0 + + def verify(parent: Path, prefix: Path, ancestors: set[Path]) -> None: + nonlocal count + resolved = parent.resolve() + if resolved in ancestors: + errors.append(f"recursive submodule cycle reaches {prefix}") + return + descendants = {*ancestors, resolved} + + for path, expected in recorded_gitlinks(parent): + display = prefix / path + checkout = parent / path + count += 1 + if not is_initialized(checkout): + errors.append(f"{display}: not initialized") + continue + actual = git( + checkout, + "rev-parse", + "HEAD", + action=f"Unable to inspect {display}", + ) + if actual != expected: + errors.append( + f"{display}: expected {expected}, found {actual}" + ) + if is_shallow(checkout): + errors.append(f"{display}: repository is still shallow") + verify(checkout, display, descendants) + + verify(repository, Path(), set()) + if errors: + details = "\n".join(f" - {error}" for error in errors) + raise CheckoutError(f"Submodule verification failed:\n{details}") + return count + + +def checkout_submodules(repository: Path) -> int: + """Prepare and verify a complete recursive development checkout.""" + root = repository_root(repository) + synchronize_submodules(root) + unshallow_initialized(root) + update_submodules(root) + unshallow_initialized(root) + synchronize_submodules(root) + update_submodules(root) + count = verify_submodules(root) + print(f"Verified {count} recursive submodule gitlinks") + return count + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "repository", + nargs="?", + type=Path, + default=Path(__file__).resolve().parents[1], + help="parent repository to prepare (default: this project)", + ) + args = parser.parse_args() + try: + checkout_submodules(args.repository) + except CheckoutError as error: + print(f"checkout-submodules: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/con_assets.py b/tools/con_assets.py new file mode 100644 index 0000000..a0336e3 --- /dev/null +++ b/tools/con_assets.py @@ -0,0 +1,1157 @@ +#!/usr/bin/env python3 +"""Hydrate and materialize assets declared by the CON profile.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import mimetypes +import os +from pathlib import Path, PurePosixPath +import re +import secrets +import shutil +import stat +import subprocess +import sys +from typing import Any, Mapping, Sequence +from urllib.error import URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen +import xml.etree.ElementTree as ET + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SITE = Path( + os.environ.get( + "CON_SITE_ROOT", + ROOT / "submodules" / "centerforopenneuroscience.org", + ) +).resolve() +UPSTREAM = Path( + os.environ.get( + "UPSTREAM_SITE_ROOT", + ROOT / "submodules" / "www-from-model", + ) +).resolve() +ASSET_MANIFEST = SITE / "profiles" / "con" / "assets.yaml" +BASELINE_MANIFEST = ROOT / "provenance" / "full-con-migration" / "baseline.yaml" +UPSTREAM_BASELINE_MANIFEST = ( + ROOT / "provenance" / "upstream-psychoinformatics" / "baseline.yaml" +) +CACHE = ROOT / "build" / "con-assets" +EXPECTED_ANNEX_VERSION = "10.20260601" +ANNEX_BUILD_NAME = "CON static build" +ANNEX_BUILD_EMAIL = "con-static-build@example.invalid" +ASSET_PREFIX = PurePosixPath("profiles/con/assets") +LINK_PREFIXES = { + "projection_links": PurePosixPath("profiles/con/projection/content"), + "static_links": PurePosixPath("profiles/con/static"), +} +SHA256 = re.compile(r"^[0-9a-f]{64}$") +MD5 = re.compile(r"^[0-9a-f]{32}$") +GIT_COMMIT = re.compile(r"^[0-9a-f]{40}$") +MEDIA_TYPE = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*/" r"[A-Za-z0-9][A-Za-z0-9!#$&^_.+-]*$" +) +ANNEX_KEY = re.compile( + r"^(?PMD5E|SHA256E)-s(?P[0-9]+)--" + r"(?P[0-9a-f]+)(?P\..+)?$" +) + + +class AssetError(RuntimeError): + """Report a manifest, retrieval, or integrity failure.""" + + +@dataclass(frozen=True) +class AssetSpec: + """A validated asset entry keyed by its site-relative destination.""" + + destination: str + source_repository: str + source_commit: str + source_path: str + availability: str + storage: str + media_type: str + mode: int + sha256: str + size: int | None + md5: str | None + annex_key: str | None + retrieval: Mapping[str, str] | None + role: str | None + + +@dataclass(frozen=True) +class GitIndexEntry: + """One stage-zero asset entry from the site Git index.""" + + mode: str + object_id: str + path: str + + +def run( + arguments: Sequence[str | Path], + *, + action: str, + check: bool = True, + cwd: Path = ROOT, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [str(argument) for argument in arguments], + cwd=cwd, + capture_output=True, + text=True, + ) + if check and result.returncode: + detail = (result.stderr or result.stdout).strip() + raise AssetError(f"{action} failed ({result.returncode}): {detail}") + return result + + +def git(repository: Path, *arguments: str, action: str) -> str: + return run(["git", "-C", repository, *arguments], action=action).stdout.strip() + + +def annex_command(repository: Path, *arguments: str) -> list[str]: + """Build a Pixi-scoped annex command for one explicit worktree.""" + git_dir = git( + repository, + "rev-parse", + "--absolute-git-dir", + action="Locate the git-annex repository", + ) + return [ + "pixi", + "run", + "git", + "-c", + f"user.name={ANNEX_BUILD_NAME}", + "-c", + f"user.email={ANNEX_BUILD_EMAIL}", + f"--git-dir={git_dir}", + f"--work-tree={repository.resolve()}", + "annex", + *arguments, + ] + + +def annex( + repository: Path, + *arguments: str, + action: str, + check: bool = True, +) -> str: + return run( + annex_command(repository, *arguments), + action=action, + check=check, + ).stdout.strip() + + +def annex_from_url( + repository: Path, + name: str, + url: str, + *arguments: str, + action: str, +) -> str: + """Use a read-only annex transport without adding a shared remote.""" + url = validate_git_repository_url( + f"Temporary annex remote {name!r}", + url, + ) + command = annex_command(repository) + command[3:3] = [ + "-c", + f"remote.{name}.url={url}", + "-c", + f"remote.{name}.fetch=+refs/heads/*:refs/remotes/{name}/*", + ] + command.extend(arguments) + return run(command, action=action).stdout.strip() + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.is_file(): + raise AssetError(f"Asset manifest is absent: {path}") + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise AssetError(f"Asset manifest must be a mapping: {path}") + return value + + +def normalized_relative_path(value: object, *, label: str) -> PurePosixPath: + if not isinstance(value, str) or not value: + raise AssetError(f"{label} must be a non-empty relative path") + path = PurePosixPath(value) + if ( + value.startswith("/") + or "\\" in value + or ".." in path.parts + or path.as_posix() != value + ): + raise AssetError(f"{label} is not a normalized relative path: {value!r}") + return path + + +def parse_mode(value: object, *, destination: str) -> int: + if value != "0644": + raise AssetError(f"{destination}: asset mode must be the string '0644'") + return int(value, 8) + + +def validate_https_url(label: str, value: object) -> str: + """Validate one credential-free HTTPS URL.""" + if ( + not isinstance(value, str) + or not value + or any(character.isspace() for character in value) + ): + raise AssetError(f"{label} must be a credential-free HTTPS URL") + try: + parsed = urlsplit(value) + hostname = parsed.hostname + parsed.port + except ValueError as error: + raise AssetError(f"{label} must be a credential-free HTTPS URL") from error + if ( + parsed.scheme != "https" + or not parsed.netloc + or hostname is None + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise AssetError(f"{label} must be a credential-free HTTPS URL") + return value + + +def validate_git_repository_url(label: str, value: object) -> str: + """Validate an HTTPS Git transport without embedded credentials.""" + return validate_https_url(f"{label} Git repository", value) + + +def validate_retrieval( + destination: str, + value: object, +) -> dict[str, str]: + if not isinstance(value, dict): + raise AssetError(f"{destination}: annex retrieval must be a mapping") + required = {"remote", "repository", "object_url", "mode"} + if set(value) != required or not all( + isinstance(value.get(key), str) and value[key] for key in required + ): + raise AssetError( + f"{destination}: annex retrieval must declare exactly {sorted(required)!r}" + ) + retrieval = {key: value[key] for key in required} + if retrieval["mode"] != "read-only": + raise AssetError(f"{destination}: annex retrieval must be read-only") + repository_value = validate_git_repository_url( + f"{destination} retrieval", retrieval["repository"] + ) + object_url_value = validate_https_url( + f"{destination} object URL", retrieval["object_url"] + ) + repository = urlsplit(repository_value) + object_url = urlsplit(object_url_value) + if (repository.scheme, repository.netloc) != ( + object_url.scheme, + object_url.netloc, + ) or not object_url.path.startswith(repository.path.rstrip("/") + "/"): + raise AssetError( + f"{destination}: object URL is outside its HTTPS read-only remote" + ) + return retrieval + + +def validate_source_repository(destination: str, value: object) -> str: + return validate_git_repository_url( + f"{destination}: source_repository", + value, + ) + + +def parse_annex_key(key: object, *, label: str) -> tuple[str, int, str]: + """Return the hash algorithm, payload size, and digest for a safe key.""" + if not isinstance(key, str): + raise AssetError(f"{label}: annex key is invalid") + match = ANNEX_KEY.fullmatch(key) + if match is None: + raise AssetError( + f"{label}: only canonical MD5E and SHA256E annex keys are supported" + ) + backend = match.group("backend") + algorithm = {"MD5E": "md5", "SHA256E": "sha256"}[backend] + digest = match.group("digest") + expected_digest = MD5 if algorithm == "md5" else SHA256 + if expected_digest.fullmatch(digest) is None: + raise AssetError(f"{label}: annex key digest is invalid") + return algorithm, int(match.group("size")), digest + + +def validate_fallback_policy(manifest: Mapping[str, Any]) -> None: + """Require an explicit no-broken-image policy for unavailable evidence.""" + expected = { + "mode": "upstream-neutral", + "person": "meerkat-person", + "project": "meerkat-project", + "render_image": True, + } + if manifest.get("fallback_policy") != expected: + raise AssetError( + "fallback_policy must declare the upstream neutral depiction behavior" + ) + omissions = manifest.get("omissions") + if not isinstance(omissions, dict): + raise AssetError("omissions must be a PID-to-policy mapping") + projection_links = manifest.get("projection_links", {}) + if not isinstance(projection_links, dict): + raise AssetError("projection_links must be a mapping") + for pid, raw in sorted(omissions.items()): + if not isinstance(pid, str) or not isinstance(raw, dict): + raise AssetError("Every asset omission must map one PID to a policy") + kind = raw.get("kind") + availability = raw.get("availability") + fallback = raw.get("fallback") + if kind == "portrait": + prefix = "xyzrins:persons/" + expected_fallback = "meerkat-person" + route = f"persons/{pid.removeprefix(prefix)}/portrait." + elif kind == "logo": + prefix = "xyzrins:projects/" + expected_fallback = "meerkat-project" + route = f"projects/{pid.removeprefix(prefix)}/logo." + else: + raise AssetError(f"{pid}: omission kind must be portrait or logo") + if not pid.startswith(prefix): + raise AssetError(f"{pid}: omission PID and kind disagree") + if availability not in {"unavailable", "absent-in-source"}: + raise AssetError(f"{pid}: omission availability is invalid") + if raw.get("projection_link", object()) is not None: + raise AssetError(f"{pid}: omitted assets must declare no projection link") + if fallback != expected_fallback: + raise AssetError(f"{pid}: omission fallback disagrees with policy") + if any(route in destination for destination in projection_links): + raise AssetError(f"{pid}: omitted asset has a projection link") + if availability == "unavailable": + source = normalized_relative_path( + raw.get("source_path"), label=f"{pid} omitted source_path" + ) + key = raw.get("annex_key") + size = raw.get("expected_size") + if not source.parts or not isinstance(key, str) or "--" not in key: + raise AssetError(f"{pid}: unavailable annex evidence is incomplete") + if not isinstance(size, int) or size < 0 or f"-s{size}--" not in key: + raise AssetError(f"{pid}: unavailable annex size/key disagree") + elif any( + field in raw for field in ("source_path", "annex_key", "expected_size") + ): + raise AssetError( + f"{pid}: absent-in-source omission cannot claim source payload fields" + ) + + +def asset_specs(manifest: Mapping[str, Any]) -> dict[str, AssetSpec]: + validate_fallback_policy(manifest) + raw_assets = manifest.get("assets") + if not isinstance(raw_assets, dict) or not raw_assets: + raise AssetError("CON asset entries must be a non-empty mapping") + specs: dict[str, AssetSpec] = {} + for destination, raw in sorted(raw_assets.items()): + path = normalized_relative_path(destination, label="Asset destination") + if ASSET_PREFIX not in (path, *path.parents): + raise AssetError( + f"Asset destination is outside {ASSET_PREFIX}: {destination}" + ) + if not isinstance(raw, dict): + raise AssetError(f"{destination}: asset entry must be a mapping") + source_repository = validate_source_repository( + destination, raw.get("source_repository") + ) + source_commit = raw.get("source_commit") + if not isinstance(source_commit, str) or not GIT_COMMIT.fullmatch( + source_commit + ): + raise AssetError(f"{destination}: source_commit must be a full Git ID") + source_path = normalized_relative_path( + raw.get("source_path"), label=f"{destination} source_path" + ).as_posix() + if raw.get("availability") != "available": + raise AssetError(f"{destination}: declared assets must be available") + storage = raw.get("storage") + if storage not in {"git", "git-annex"}: + raise AssetError(f"{destination}: storage must be 'git' or 'git-annex'") + media_type = raw.get("media_type") + if not isinstance(media_type, str) or not MEDIA_TYPE.fullmatch(media_type): + raise AssetError(f"{destination}: media_type is invalid") + sha256 = raw.get("sha256") + if not isinstance(sha256, str) or not SHA256.fullmatch(sha256): + raise AssetError(f"{destination}: sha256 digest is invalid") + size = raw.get("size") + if not isinstance(size, int) or size < 0: + raise AssetError(f"{destination}: size must be a non-negative integer") + md5 = raw.get("md5") + if md5 is not None and (not isinstance(md5, str) or not MD5.fullmatch(md5)): + raise AssetError(f"{destination}: md5 digest is invalid") + mode = parse_mode(raw.get("mode"), destination=destination) + annex_key = raw.get("annex_key") + retrieval = raw.get("retrieval") + if storage == "git-annex": + algorithm, key_size, key_digest = parse_annex_key( + annex_key, + label=destination, + ) + retrieval = validate_retrieval(destination, retrieval) + if size != key_size: + raise AssetError(f"{destination}: annex key and declared size disagree") + if algorithm == "md5" and md5 != key_digest: + raise AssetError(f"{destination}: annex key and md5 disagree") + if algorithm == "sha256" and sha256 != key_digest: + raise AssetError(f"{destination}: annex key and sha256 disagree") + elif annex_key is not None or retrieval is not None: + raise AssetError(f"{destination}: ordinary Git asset has annex-only fields") + role = raw.get("role") + if role is not None and (not isinstance(role, str) or not role): + raise AssetError(f"{destination}: role must be a non-empty string") + specs[destination] = AssetSpec( + destination=destination, + source_repository=source_repository, + source_commit=source_commit, + source_path=source_path, + availability="available", + storage=storage, + media_type=media_type, + mode=mode, + sha256=sha256, + size=size, + md5=md5, + annex_key=annex_key, + retrieval=retrieval, + role=role, + ) + return specs + + +def verify_annex_runtime() -> None: + """Require the pinned git-annex executable from the Pixi environment.""" + baseline = load_yaml(BASELINE_MANIFEST) + toolchain = baseline.get("toolchain", {}) + expected = toolchain.get("git_annex") if isinstance(toolchain, dict) else None + if expected != EXPECTED_ANNEX_VERSION: + raise AssetError( + f"Expected full-migration provenance to pin git-annex " + f"{EXPECTED_ANNEX_VERSION}, found {expected!r}" + ) + executable = run( + [ + "pixi", + "run", + "python", + "-c", + "import shutil; print(shutil.which('git-annex') or '')", + ], + action="Locate the Pixi git-annex runtime", + ).stdout.strip() + environment_root = (ROOT / ".pixi" / "envs").resolve() + try: + executable_path = Path(executable).resolve() + executable_path.relative_to(environment_root) + except (OSError, ValueError) as error: + raise AssetError( + f"git-annex is not provided by this project's Pixi environment: " + f"{executable!r}" + ) from error + output = run( + ["pixi", "run", "git", "annex", "version"], + action="Inspect the Pixi git-annex runtime", + ).stdout + first_line = output.splitlines()[0] if output else "" + actual = first_line.removeprefix("git-annex version: ").split("-", 1)[0] + if actual != expected: + raise AssetError(f"Expected git-annex {expected}, found {first_line!r}") + + +def upstream_annex_entries() -> dict[str, str]: + baseline = load_yaml(UPSTREAM_BASELINE_MANIFEST) + annex_info = baseline.get("annex", {}) + entries = ( + annex_info.get("upstream_annex_only") if isinstance(annex_info, dict) else None + ) + if not isinstance(entries, list): + raise AssetError("Upstream annex inventory is incomplete") + result: dict[str, str] = {} + for entry in entries: + if not isinstance(entry, dict): + raise AssetError("Invalid upstream annex inventory entry") + path = normalized_relative_path( + entry.get("path"), label="Upstream annex path" + ).as_posix() + key = entry.get("key") + if not isinstance(key, str) or not key: + raise AssetError(f"Upstream annex key is invalid for {path}") + if path.startswith(("assets/", "static/")): + result[path] = key + if not result: + raise AssetError("Upstream annex inventory declares no Hugo assets") + return result + + +def annex_path_available(repository: Path, path: str) -> bool: + present = annex( + repository, + "find", + path, + "--in=here", + action=f"Inspect annex availability for {path}", + ) + return bool(present) + + +def temporary_remote_config(repository: Path, name: str) -> str: + pattern = rf"^remote\.{re.escape(name)}\." + result = run( + ["git", "-C", repository, "config", "--local", "--get-regexp", pattern], + action="Inspect temporary annex remote configuration", + check=False, + ) + if result.returncode not in {0, 1}: + detail = (result.stderr or result.stdout).strip() + raise AssetError(f"Could not inspect temporary remote config: {detail}") + return result.stdout + + +def remove_temporary_remote_config(repository: Path, name: str) -> None: + """Remove only metadata git-annex inferred for our ephemeral remote.""" + result = run( + [ + "git", + "-C", + repository, + "config", + "--local", + "--remove-section", + f"remote.{name}", + ], + action="Remove temporary annex remote configuration", + check=False, + ) + if result.returncode not in {0, 5}: + detail = (result.stderr or result.stdout).strip() + raise AssetError(f"Could not remove temporary remote config: {detail}") + + +def hydrate_upstream() -> None: + """Hydrate declared upstream Hugo pointers through a temporary transport.""" + baseline = load_yaml(UPSTREAM_BASELINE_MANIFEST) + website = baseline.get("website", {}) + if not isinstance(website, dict): + raise AssetError("Invalid upstream annex baseline manifest") + commit = website.get("annex_metadata_commit") + if not isinstance(commit, str) or not GIT_COMMIT.fullmatch(commit): + raise AssetError("Upstream annex provenance is incomplete") + url = validate_git_repository_url( + "Upstream annex metadata", + website.get("upstream_repository"), + ) + entries = upstream_annex_entries() + + annex(UPSTREAM, "init", action="Initialize upstream annex") + for path, expected_key in entries.items(): + actual_key = annex( + UPSTREAM, + "lookupkey", + path, + action=f"Inspect upstream annex key for {path}", + ) + if actual_key != expected_key: + raise AssetError( + f"Upstream annex key changed for {path}: " + f"expected {expected_key}, found {actual_key!r}" + ) + missing = [ + path for path in sorted(entries) if not annex_path_available(UPSTREAM, path) + ] + if missing: + name = "full-con-migration-upstream" + remote_ref = f"refs/remotes/{name}/git-annex" + before = temporary_remote_config(UPSTREAM, name) + if before: + raise AssetError(f"Temporary annex remote already exists: {name}") + try: + git( + UPSTREAM, + "fetch", + "--no-write-fetch-head", + "--depth", + "1", + url, + f"+{commit}:{remote_ref}", + action="Fetch pinned upstream annex metadata", + ) + annex_from_url( + UPSTREAM, + name, + url, + "get", + "--from", + name, + "--", + *missing, + action="Hydrate upstream Hugo assets", + ) + finally: + try: + git( + UPSTREAM, + "update-ref", + "-d", + remote_ref, + action="Remove temporary upstream annex ref", + ) + finally: + remove_temporary_remote_config(UPSTREAM, name) + after = temporary_remote_config(UPSTREAM, name) + if after: + raise AssetError(f"Annex hydration persisted temporary remote {name!r}") + unavailable = [ + path for path in sorted(entries) if not annex_path_available(UPSTREAM, path) + ] + if unavailable: + raise AssetError( + "Required upstream annex paths remain unavailable:\n" + + "\n".join(unavailable) + ) + for path, key in sorted(entries.items()): + verify_payload_against_annex_key( + UPSTREAM / path, + key, + label=f"Upstream annex payload {path}", + ) + + +def file_digest(path: Path, algorithm: str) -> str: + hasher = hashlib.new(algorithm) + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def detected_media_type(path: Path) -> str: + with path.open("rb") as stream: + header = stream.read(65536) + if header.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if header.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if header.startswith((b"GIF87a", b"GIF89a")): + return "image/gif" + if header.startswith(b"RIFF") and header[8:12] == b"WEBP": + return "image/webp" + if header.startswith(b"%PDF-"): + return "application/pdf" + if header.startswith(b"\x00\x00\x01\x00"): + return "image/vnd.microsoft.icon" + guessed = mimetypes.guess_type(path.name)[0] + if guessed == "image/svg+xml": + try: + root = ET.parse(path).getroot() + except (ET.ParseError, OSError) as error: + raise AssetError(f"{path}: invalid SVG content") from error + if root.tag.rsplit("}", 1)[-1] != "svg": + raise AssetError(f"{path}: XML root is not SVG") + return guessed + if guessed and guessed.startswith("text/"): + try: + path.read_text(encoding="utf-8") + except UnicodeDecodeError as error: + raise AssetError(f"{path}: declared text asset is not UTF-8") from error + return guessed + raise AssetError(f"{path}: unsupported or unrecognized asset media type") + + +def verify_file(path: Path, spec: AssetSpec) -> None: + if not path.is_file(): + raise AssetError(f"Required asset is absent: {path}") + actual_mode = stat.S_IMODE(path.stat().st_mode) + if actual_mode != spec.mode: + raise AssetError( + f"{path}: expected mode {spec.mode:04o}, found {actual_mode:04o}" + ) + if spec.size is not None and path.stat().st_size != spec.size: + raise AssetError( + f"{path}: expected {spec.size} bytes, found {path.stat().st_size}" + ) + if file_digest(path, "sha256") != spec.sha256: + raise AssetError(f"{path}: sha256 digest does not match") + if spec.md5 and file_digest(path, "md5") != spec.md5: + raise AssetError(f"{path}: md5 digest does not match") + actual_media_type = detected_media_type(path) + if actual_media_type != spec.media_type: + raise AssetError( + f"{path}: expected media type {spec.media_type}, found {actual_media_type}" + ) + + +def git_index_entry(repository: Path, destination: str) -> GitIndexEntry: + """Read exactly one stage-zero entry for an asset destination.""" + result = run( + [ + "git", + "-C", + repository, + "ls-files", + "--stage", + "-z", + "--", + destination, + ], + action=f"Inspect the Git index entry for {destination}", + check=False, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + raise AssetError(f"Could not inspect the Git index for {destination}: {detail}") + records = [record for record in result.stdout.split("\0") if record] + if len(records) != 1 or "\t" not in records[0]: + raise AssetError(f"{destination}: asset must have exactly one Git index entry") + header, indexed_path = records[0].split("\t", 1) + fields = header.split() + if len(fields) != 3 or fields[2] != "0" or indexed_path != destination: + raise AssetError(f"{destination}: Git index entry is not stage zero") + mode, object_id, _stage = fields + return GitIndexEntry(mode=mode, object_id=object_id, path=indexed_path) + + +def annex_hashdir(key: str) -> PurePosixPath: + """Return the canonical mixed-hash object directory for an annex key.""" + value = annex( + SITE, + "examinekey", + "--format=${hashdirmixed}", + key, + action=f"Calculate the annex object directory for {key}", + ) + if not value.endswith("/"): + raise AssetError(f"{key}: git-annex returned an invalid object directory") + path = normalized_relative_path( + value.removesuffix("/"), + label=f"{key} annex object directory", + ) + if len(path.parts) != 2: + raise AssetError(f"{key}: git-annex returned an invalid object directory") + return path + + +def canonical_annex_pointer_target(spec: AssetSpec) -> str: + """Calculate the portable symlink text committed for an annex asset.""" + if not spec.annex_key: + raise AssetError(f"{spec.destination}: annex key is absent") + pointer = SITE.joinpath(*PurePosixPath(spec.destination).parts) + object_path = ( + SITE + / ".git" + / "annex" + / "objects" + / Path(*annex_hashdir(spec.annex_key).parts) + / spec.annex_key + / spec.annex_key + ) + return Path(os.path.relpath(object_path, pointer.parent)).as_posix() + + +def verify_git_index_contract(spec: AssetSpec) -> str | None: + """Require the committed representation promised by the manifest.""" + entry = git_index_entry(SITE, spec.destination) + expected_mode = "100644" if spec.storage == "git" else "120000" + if entry.mode != expected_mode: + raise AssetError( + f"{spec.destination}: expected Git index mode {expected_mode}, " + f"found {entry.mode}" + ) + if spec.storage == "git": + return None + target = canonical_annex_pointer_target(spec) + indexed_target = run( + ["git", "-C", SITE, "cat-file", "blob", entry.object_id], + action=f"Inspect the indexed annex pointer for {spec.destination}", + ).stdout + if indexed_target != target: + raise AssetError(f"{spec.destination}: indexed annex pointer is not canonical") + return target + + +def verify_annex_pointer( + spec: AssetSpec, + expected_target: str | None = None, +) -> None: + path = SITE.joinpath(*PurePosixPath(spec.destination).parts) + if not path.is_symlink(): + raise AssetError(f"{spec.destination}: annex asset is not a symlink") + target = os.readlink(path) + expected = expected_target or canonical_annex_pointer_target(spec) + if target != expected: + raise AssetError( + f"{spec.destination}: working-tree annex pointer is not canonical" + ) + + +def verify_payload_against_annex_key( + path: Path, + key: str, + *, + label: str, +) -> None: + """Verify payload bytes and the Pixi git-annex calculation for one key.""" + algorithm, expected_size, expected_digest = parse_annex_key(key, label=label) + if not path.is_file(): + raise AssetError(f"{label}: payload is absent: {path}") + actual_size = path.stat().st_size + if actual_size != expected_size: + raise AssetError( + f"{label}: expected {expected_size} bytes, found {actual_size}" + ) + if file_digest(path, algorithm) != expected_digest: + raise AssetError(f"{label}: {algorithm} digest does not match its annex key") + backend = key.split("-", 1)[0] + actual = run( + [ + "pixi", + "run", + "git", + "annex", + "calckey", + f"--backend={backend}", + path, + ], + action=f"Validate annex key for {label}", + ).stdout.strip() + if actual != key: + raise AssetError(f"{label}: expected annex key {key}, found {actual!r}") + + +def verify_annex_key(path: Path, spec: AssetSpec) -> None: + if not spec.annex_key: + raise AssetError(f"{spec.destination}: annex key is absent") + verify_payload_against_annex_key( + path, + spec.annex_key, + label=spec.destination, + ) + + +def cache_path(spec: AssetSpec) -> Path: + root = Path(os.path.abspath(CACHE)) + return root.joinpath(*PurePosixPath(spec.destination).parts) + + +def ensure_safe_directory_chain(root: Path, parent: Path, *, label: str) -> None: + """Create a directory chain without accepting symlinked components.""" + root = Path(os.path.abspath(root)) + parent = Path(os.path.abspath(parent)) + try: + relative = parent.relative_to(root) + except ValueError as error: + raise AssetError(f"{label}: destination escapes its root") from error + if root.is_symlink(): + raise AssetError(f"{label}: root directory is a symlink: {root}") + root.mkdir(parents=True, exist_ok=True) + root_status = os.lstat(root) + if stat.S_ISLNK(root_status.st_mode) or not stat.S_ISDIR(root_status.st_mode): + raise AssetError(f"{label}: root is not a safe directory: {root}") + current = root + for part in relative.parts: + current = current / part + try: + current_status = os.lstat(current) + except FileNotFoundError: + try: + os.mkdir(current, 0o755) + except FileExistsError: + current_status = os.lstat(current) + else: + current_status = os.lstat(current) + if stat.S_ISLNK(current_status.st_mode): + raise AssetError(f"{label}: directory ancestor is a symlink: {current}") + if not stat.S_ISDIR(current_status.st_mode): + raise AssetError(f"{label}: ancestor is not a directory: {current}") + + +def inspect_safe_cache_destination(destination: Path) -> bool: + """Prepare cache parents and report whether a safe regular file exists.""" + root = Path(os.path.abspath(CACHE)) + ensure_safe_directory_chain(root, destination.parent, label="Asset cache") + try: + destination_status = os.lstat(destination) + except FileNotFoundError: + return False + if stat.S_ISLNK(destination_status.st_mode): + raise AssetError(f"Asset cache destination is a symlink: {destination}") + if not stat.S_ISREG(destination_status.st_mode): + raise AssetError(f"Asset cache destination is not a file: {destination}") + return True + + +def open_exclusive_download(path: Path) -> int: + """Open a new temporary payload without following or replacing a path.""" + no_follow = getattr(os, "O_NOFOLLOW", 0) + if not no_follow: + raise AssetError("This platform cannot create no-follow asset downloads") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | no_follow + flags |= getattr(os, "O_CLOEXEC", 0) + try: + return os.open(path, flags, 0o600) + except OSError as error: + raise AssetError( + f"Could not create exclusive asset download {path}: {error}" + ) from error + + +def hydrate_annex_asset(spec: AssetSpec) -> Path: + if spec.retrieval is None or spec.annex_key is None: + raise AssetError(f"{spec.destination}: annex retrieval is incomplete") + destination = cache_path(spec) + if inspect_safe_cache_destination(destination): + verify_file(destination, spec) + verify_annex_key(destination, spec) + return destination + + temporary = destination.with_name( + f".download-{os.getpid()}-{secrets.token_hex(8)}-{destination.name}" + ) + request = Request( + spec.retrieval["object_url"], + headers={"User-Agent": "full-con-migration/1"}, + ) + temporary_created = False + try: + descriptor = open_exclusive_download(temporary) + temporary_created = True + with ( + os.fdopen( + descriptor, + "wb", + ) as out, + urlopen(request, timeout=120) as response, + ): + shutil.copyfileobj(response, out) + os.fchmod(out.fileno(), spec.mode) + verify_file(temporary, spec) + verify_annex_key(temporary, spec) + inspect_safe_cache_destination(destination) + os.replace(temporary, destination) + except (OSError, URLError) as error: + raise AssetError( + f"Could not hydrate annex key {spec.annex_key} from " + f"{spec.retrieval['remote']}: {error}" + ) from error + finally: + if temporary_created: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + return destination + + +def verify_git_assets(specs: Mapping[str, AssetSpec]) -> dict[str, Path]: + files: dict[str, Path] = {} + for destination, spec in sorted(specs.items()): + if spec.storage != "git": + continue + path = SITE.joinpath(*PurePosixPath(destination).parts) + verify_git_index_contract(spec) + if path.is_symlink(): + raise AssetError(f"{destination}: ordinary Git asset is a symlink") + verify_file(path, spec) + files[destination] = path + return files + + +def hydrate_manifest_assets( + manifest: Mapping[str, Any], +) -> dict[str, Path]: + """Validate every entry and return its materialized local file.""" + specs = asset_specs(manifest) + files = verify_git_assets(specs) + for destination, spec in sorted(specs.items()): + if spec.storage != "git-annex": + continue + target = verify_git_index_contract(spec) + verify_annex_pointer(spec, target) + files[destination] = hydrate_annex_asset(spec) + if set(files) != set(specs): + raise AssetError("Not every declared asset was materialized") + return files + + +def materialization_plan( + manifest: Mapping[str, Any], + specs: Mapping[str, AssetSpec], +) -> dict[str, str]: + """Validate all projection/static destinations and their declared sources.""" + plan: dict[str, str] = {} + for group, prefix in LINK_PREFIXES.items(): + links = manifest.get(group, {}) + if not isinstance(links, dict): + raise AssetError(f"{group} must be a destination-to-source mapping") + for destination, source in sorted(links.items()): + destination_path = normalized_relative_path( + destination, label=f"{group} destination" + ) + if prefix not in (destination_path, *destination_path.parents): + raise AssetError( + f"{group} destination is outside {prefix}: {destination}" + ) + source_path = normalized_relative_path( + source, label=f"{group} source" + ).as_posix() + if source_path not in specs: + raise AssetError( + f"{group} source is not a declared asset: {source_path}" + ) + if destination in plan: + raise AssetError( + f"Materialization destination is declared twice: {destination}" + ) + guessed = mimetypes.guess_type(destination_path.name)[0] + expected = specs[source_path].media_type + if guessed != expected: + raise AssetError( + f"{destination}: extension implies {guessed}, " + f"but source is {expected}" + ) + plan[destination] = source_path + return plan + + +def copy_materialized_file( + root: Path, + destination: str, + source: Path, + spec: AssetSpec, +) -> Path: + root = Path(os.path.abspath(root)) + target = Path(os.path.abspath(root.joinpath(*PurePosixPath(destination).parts))) + try: + target.relative_to(root) + except ValueError as error: + raise AssetError(f"Materialization escapes its root: {destination}") from error + ensure_safe_directory_chain( + root, + target.parent, + label=f"Asset materialization {destination}", + ) + try: + target_status = os.lstat(target) + except FileNotFoundError: + pass + else: + if stat.S_ISLNK(target_status.st_mode): + if destination != spec.destination or spec.storage != "git-annex": + raise AssetError(f"Materialization destination is a symlink: {target}") + actual_pointer = os.readlink(target) + expected_pointer = canonical_annex_pointer_target(spec) + if actual_pointer != expected_pointer: + raise AssetError( + "Materialization destination is not the declared canonical " + f"annex pointer: {target}" + ) + os.unlink(target) + elif not stat.S_ISREG(target_status.st_mode): + raise AssetError(f"Materialization destination is not a file: {target}") + else: + os.unlink(target) + shutil.copyfile(source, target) + target.chmod(spec.mode) + verify_file(target, spec) + return target + + +def materialize_declared_assets( + destination_root: Path, + manifest: Mapping[str, Any], + files: Mapping[str, Path], +) -> list[Path]: + """Replace every declared asset destination in an assembly tree.""" + specs = asset_specs(manifest) + root = Path(os.path.abspath(destination_root)) + materialized: list[Path] = [] + for destination, spec in sorted(specs.items()): + source = files.get(destination) + if source is None: + raise AssetError(f"Declared asset is unavailable: {destination}") + materialized.append(copy_materialized_file(root, destination, source, spec)) + return materialized + + +def materialize_declared_links( + destination_root: Path, + manifest: Mapping[str, Any], + files: Mapping[str, Path], +) -> list[Path]: + """Copy each declared projection/static link into an assembly root.""" + specs = asset_specs(manifest) + plan = materialization_plan(manifest, specs) + root = Path(os.path.abspath(destination_root)) + materialized: list[Path] = [] + for destination, source in sorted(plan.items()): + source_path = files.get(source) + if source_path is None: + raise AssetError(f"Materialized source is unavailable: {source}") + materialized.append( + copy_materialized_file( + root, + destination, + source_path, + specs[source], + ) + ) + return materialized + + +def materialize_all_assets( + destination_root: Path, + manifest: Mapping[str, Any], + files: Mapping[str, Path], +) -> list[Path]: + """Materialize all sources and declared copies into an assembly tree.""" + return [ + *materialize_declared_assets(destination_root, manifest, files), + *materialize_declared_links(destination_root, manifest, files), + ] + + +def hydrate_all_assets() -> dict[str, Path]: + """Hydrate every declared dependency and return all CON asset files.""" + verify_annex_runtime() + hydrate_upstream() + manifest = load_yaml(ASSET_MANIFEST) + specs = asset_specs(manifest) + files = hydrate_manifest_assets(manifest) + materialization_plan(manifest, specs) + return files + + +def main() -> int: + try: + manifest = load_yaml(ASSET_MANIFEST) + hydrate_all_assets() + specs = asset_specs(manifest) + print(f"Hydrated {len(specs)} manifest assets") + except AssetError as error: + print(f"full-migration assets: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/con_projection.py b/tools/con_projection.py new file mode 100644 index 0000000..42c0436 --- /dev/null +++ b/tools/con_projection.py @@ -0,0 +1,2702 @@ +#!/usr/bin/env python3 +"""Render and verify the committed clean-migration projection.""" + +from __future__ import annotations + +import argparse +import ast +from collections import Counter, defaultdict +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import time +import tomllib +from typing import Any, Iterator, Sequence +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + +from packaging.requirements import InvalidRequirement, Requirement +from packaging.utils import canonicalize_name +import yaml + +from dump_things_service import Format +from dump_things_service.converter import FormatConverter +from linkml_runtime import SchemaView + + +ROOT = Path(__file__).resolve().parents[1] +SITE = Path( + os.environ.get( + "CON_SITE_ROOT", + ROOT / "submodules" / "centerforopenneuroscience.org", + ) +).resolve() +UPSTREAM = Path( + os.environ.get( + "UPSTREAM_SITE_ROOT", + ROOT / "submodules" / "www-from-model", + ) +).resolve() +PROFILE_ROOT = SITE / "profiles" / "con" +PROFILE_PATH = PROFILE_ROOT / "profile.yaml" +PROJECTION_SPEC_PATH = PROFILE_ROOT / "projection.yaml" +COMMITTED = PROFILE_ROOT / "projection" +PROJECTION_ATTRIBUTES = COMMITTED / ".gitattributes" +SCHEMA = ( + ROOT + / "submodules" + / "things-schemas" + / "src" + / "demo-research-information" + / "unreleased.yaml" +) +BUILD_ROOT = ROOT / "build" / "con-projection" +COLLECTION = "con-public" +READER_TOKEN = "con-projection-reader" +VALIDATOR_TOKEN = "con-projection-validator" +REVIEWED_FULL_MIGRATION_BASE = "a9ac9d5abc3898fd13d9b8392008f0c323c8dcd8" +ACCEPTED_CLEAN_MIGRATION_TIP = "a122e506de9e4a13473edbe8d74a950d74032a16" +ACCEPTED_CLEAN_MIGRATION_PARENT_TIP = "f54cf5fdb2b5ae4bf03fe6939246316fd9ec818d" +FOUNDATION_SUBJECTS = ( + "build(clean-migration): add the CON site profile", + "feat(content): add the clean CON vertical slice", +) +TERMINAL_PROJECTION_SUBJECT = "chore(projection): refresh the full CON snapshot" +ASSEMBLY_DIGEST_PATH = "profiles/con/assembly/SHA256SUMS" +CONVENTIONAL_SUBJECT = re.compile( + r"^(?:build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test)" + r"(?:\([^)]+\))?!?: .+" +) +UPSTREAM_UPDATE_WORKFLOW = Path(".forgejo/workflows/update-from-pool.yaml") + +REQUIRED_NATIVE_TYPES = { + "dlthings:Association", + "dlthings:Attribution", + "dlthings:Generation", + "dlthings:DOI", + "dlthings:ISSN", +} +FORBIDDEN_BRIDGE_PREDICATES = { + "dcterms:contributor", + "dcterms:creator", + "dcterms:relation", + "schema:about", + "schema:member", + "schema:memberOf", + "schema:subjectOf", +} +PROJECTION_PYPI_ROOTS = ( + "dump-things-pyclient", + "dump-things-service", + "jinja2", + "linkml", + "linkml-runtime", + "packaging", + "pydantic", + "pyyaml", + "query-things", + "rdflib", +) +PROJECTION_LOCAL_PYPI_PATHS = { + "dump-things-pyclient": "submodules/dump-things-pyclient", + "dump-things-service": "submodules/dump-things-service", + "query-things": "submodules/query-things", +} + + +class ProjectionError(RuntimeError): + """Report a fail-closed clean-migration contract violation.""" + + +@dataclass(frozen=True) +class SourceRecord: + """One canonical or reference record and its declared top-level class.""" + + class_name: str + record: dict[str, Any] + path: Path + category: str + + +@dataclass(frozen=True) +class ProjectionContract: + """Executable record, page, and graph policy from the site manifests.""" + + canonical_root: Path + reference_root: Path + collection: str + homepage_pid: str + homepage_class: str + homepage_record: Path + homepage_template: Path + page_templates: dict[str, Path] + unrendered_classes: frozenset[str] + graph_node_classes: frozenset[str] + graph_relationship_fields: tuple[str, ...] + snapshot_path: Path + content_root: Path + graph_output: Path + digest_output: Path + + +@dataclass(frozen=True) +class ProjectionExpectations: + """Closure derived from the profile contract and its source records.""" + + canonical_pids: frozenset[str] + reference_pids: frozenset[str] + graph_node_pids: frozenset[str] + graph_edges: frozenset[tuple[str, str]] + markdown_pages: frozenset[str] + entity_routes: frozenset[str] + record_payloads: tuple[tuple[str, str], ...] + + +def run( + arguments: Sequence[str | Path], + *, + input_text: str | None = None, + environment: dict[str, str] | None = None, + cwd: Path = ROOT, + action: str, +) -> str: + """Run a command and return stdout with a useful failure message.""" + command = [str(argument) for argument in arguments] + result = subprocess.run( + command, + cwd=cwd, + env=environment, + input=input_text, + capture_output=True, + text=True, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + raise ProjectionError(f"{action} failed ({result.returncode}): {detail}") + return result.stdout + + +def git_commit(repository: Path) -> str: + return run( + ["git", "-C", repository, "rev-parse", "HEAD"], + action=f"Inspect {repository.name} checkout", + ).strip() + + +def git_tree_object(repository: Path, expression: str) -> str: + return run( + ["git", "-C", repository, "rev-parse", expression], + action=f"Inspect {repository.name} tree object {expression}", + ).strip() + + +def require_clean_checkout(repository: Path, label: str) -> None: + status = run( + [ + "git", + "-C", + repository, + "status", + "--porcelain", + "--untracked-files=all", + ], + action=f"Inspect the {label} worktree", + ).strip() + if status: + raise ProjectionError(f"The pinned {label} worktree has changes:\n{status}") + + +def require_no_ignored_files( + repository: Path, + label: str, + pathspecs: Sequence[str] = (), +) -> None: + """Reject ignored worktree files that Git's normal clean check omits.""" + command: list[str | Path] = [ + "git", + "-C", + repository, + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + ] + if pathspecs: + command.extend(["--", *pathspecs]) + ignored = run( + command, + action=f"Inspect ignored files in the {label} worktree", + ).splitlines() + if ignored: + raise ProjectionError( + f"The pinned {label} worktree has ignored files: " + + ", ".join(sorted(ignored)) + ) + + +def verify_transport_trees() -> None: + """Require the hydrated sibling to match the rebased site's payload trees.""" + for path in ("assets", "static", "themes/congo"): + site_object = git_tree_object(SITE, f"HEAD:{path}") + transport_object = git_tree_object(UPSTREAM, f"HEAD:{path}") + if site_object != transport_object: + raise ProjectionError( + "The upstream hydration checkout differs from the rebased " + f"site for {path}: {site_object} != {transport_object}" + ) + + +def allowed_site_overlay_path(path: str, *, dirty: bool) -> bool: + """Return whether one path belongs to the isolated downstream layer.""" + if path == ".gitmodules": + return True + if not dirty and path == "UPSTREAM.md": + return True + return path.startswith(("config/con/", "profiles/con/")) + + +def generated_snapshot_path(path: str) -> bool: + """Identify projection and assembly outputs reserved for the terminal commit.""" + return path.startswith("profiles/con/projection/") or path == ASSEMBLY_DIGEST_PATH + + +def site_status_paths(repository: Path) -> list[str]: + """Return every current and rename-source path from porcelain status.""" + result = subprocess.run( + [ + "git", + "-C", + str(repository), + "status", + "--porcelain=v1", + "-z", + "--untracked-files=all", + ], + capture_output=True, + text=True, + ) + if result.returncode: + detail = (result.stderr or result.stdout).strip() + raise ProjectionError(f"Inspect site worktree failed: {detail}") + entries = result.stdout.split("\0") + paths: list[str] = [] + index = 0 + while index < len(entries): + entry = entries[index] + index += 1 + if not entry: + continue + if len(entry) < 4 or entry[2] != " ": + raise ProjectionError(f"Cannot parse site status entry: {entry!r}") + status = entry[:2] + paths.append(entry[3:]) + if any(code in "RC" for code in status): + if index >= len(entries) or not entries[index]: + raise ProjectionError("Site status rename has no source path") + paths.append(entries[index]) + index += 1 + return paths + + +def verify_site_worktree_isolation(repository: Path = SITE) -> None: + """Allow dirty migration inputs, but reject upstream-owned changes.""" + unexpected = sorted( + path + for path in site_status_paths(repository) + if not allowed_site_overlay_path(path, dirty=True) + ) + if unexpected: + raise ProjectionError( + "The site worktree has dirty or untracked upstream-owned paths: " + + ", ".join(unexpected) + ) + + +def checkpoint_refs(repository: Path = SITE) -> dict[str, str]: + """Find local or remote-tracking refs for the accepted checkpoint.""" + output = run( + [ + "git", + "-C", + repository, + "for-each-ref", + "--format=%(objectname) %(refname)", + "refs/heads/codex/clean-migration", + "refs/remotes/*/codex/clean-migration", + ], + action="Inspect the accepted clean-migration checkpoint", + ) + return { + ref: commit + for line in output.splitlines() + if line + for commit, ref in [line.split(" ", 1)] + } + + +def verify_terminal_history( + terminal_indexes: Sequence[int], + commit_count: int, + *, + require_terminal: bool, +) -> None: + """Enforce preparation or final-mode generated snapshot history.""" + if len(terminal_indexes) > 1 or ( + terminal_indexes and terminal_indexes[0] != commit_count - 1 + ): + raise ProjectionError( + "The generated projection commit must be unique and terminal" + ) + if require_terminal and len(terminal_indexes) != 1: + raise ProjectionError( + "Final acceptance requires exactly one terminal generated " + f"projection commit, found {len(terminal_indexes)}" + ) + + +def verify_linear_successor_history( + repository: Path, + base: str, + commits: Sequence[str], +) -> None: + """Require every successor commit to have exactly its predecessor as parent.""" + expected_parent = base + for commit in commits: + parents = run( + ["git", "-C", repository, "show", "-s", "--format=%P", commit], + action=f"Inspect successor parents for {commit}", + ).split() + if parents != [expected_parent]: + raise ProjectionError( + "The successor site history must be a linear commit stack: " + f"{commit} has parents {parents}, expected {[expected_parent]}" + ) + expected_parent = commit + + +def verify_successor_history( + profile: dict[str, Any], + repository: Path = SITE, + *, + require_terminal: bool = False, +) -> None: + """Validate focused successor commits without reviving the two-commit rule.""" + components = profile.get("components") + if not isinstance(components, dict): + raise ProjectionError("Profile components must be a mapping") + website = components.get("www_from_model") + if not isinstance(website, dict): + raise ProjectionError("Profile www_from_model component must be a mapping") + declared_upstream = website.get("commit") + if declared_upstream != REVIEWED_FULL_MIGRATION_BASE: + raise ProjectionError( + "The full-migration profile must use the reviewed upstream base " + f"{REVIEWED_FULL_MIGRATION_BASE}, found {declared_upstream!r}" + ) + ancestor = subprocess.run( + [ + "git", + "-C", + str(repository), + "merge-base", + "--is-ancestor", + declared_upstream, + "HEAD", + ], + capture_output=True, + text=True, + ) + if ancestor.returncode: + raise ProjectionError( + "The reviewed upstream commit is not an ancestor of the " + f"full-migration site: {declared_upstream}" + ) + + commits = run( + [ + "git", + "-C", + repository, + "rev-list", + "--reverse", + f"{declared_upstream}..HEAD", + ], + action="Inspect full-migration website commits", + ).splitlines() + if len(commits) < len(FOUNDATION_SUBJECTS): + raise ProjectionError( + "The successor site is missing its accepted foundation commits" + ) + verify_linear_successor_history(repository, declared_upstream, commits) + subjects = [ + run( + ["git", "-C", repository, "show", "-s", "--format=%s", commit], + action=f"Inspect successor commit {commit}", + ).strip() + for commit in commits + ] + if tuple(subjects[:2]) != FOUNDATION_SUBJECTS: + raise ProjectionError( + f"The successor foundation commit subjects differ: {subjects[:2]}" + ) + unconventional = [ + subject for subject in subjects if not CONVENTIONAL_SUBJECT.fullmatch(subject) + ] + if unconventional: + raise ProjectionError( + f"Successor site commits must be Conventional Commits: {unconventional}" + ) + + terminal_indexes = [ + index + for index, subject in enumerate(subjects) + if subject == TERMINAL_PROJECTION_SUBJECT + ] + verify_terminal_history( + terminal_indexes, + len(commits), + require_terminal=require_terminal, + ) + for index, commit in enumerate(commits): + paths = run( + [ + "git", + "-C", + repository, + "diff-tree", + "--no-commit-id", + "--name-only", + "-r", + commit, + ], + action=f"Inspect successor paths in {commit}", + ).splitlines() + unexpected = [ + path for path in paths if not allowed_site_overlay_path(path, dirty=False) + ] + if unexpected: + raise ProjectionError( + f"Successor commit {commit} changes upstream-owned paths: " + + ", ".join(unexpected) + ) + if index < len(FOUNDATION_SUBJECTS): + continue + generated = [path for path in paths if generated_snapshot_path(path)] + if subjects[index] == TERMINAL_PROJECTION_SUBJECT: + if not generated: + raise ProjectionError( + "The terminal projection commit contains no generated outputs" + ) + if len(generated) != len(paths): + raise ProjectionError( + "The terminal projection commit contains hand-authored paths" + ) + elif generated: + raise ProjectionError( + f"Hand-authored successor commit {commit} contains generated " + "projection paths" + ) + + refs = checkpoint_refs(repository) + if ACCEPTED_CLEAN_MIGRATION_TIP not in refs.values(): + raise ProjectionError( + "No clean-migration checkpoint ref preserves accepted site tip " + f"{ACCEPTED_CLEAN_MIGRATION_TIP}: {refs}" + ) + parent_refs = checkpoint_refs(ROOT) + if ACCEPTED_CLEAN_MIGRATION_PARENT_TIP not in parent_refs.values(): + raise ProjectionError( + "No clean-migration checkpoint ref preserves accepted parent tip " + f"{ACCEPTED_CLEAN_MIGRATION_PARENT_TIP}: {parent_refs}" + ) + + +def verify_final_site_state( + profile: dict[str, Any] | None = None, + repository: Path = SITE, +) -> None: + """Require the immutable, clean site state used by final acceptance.""" + profile = ( + load_yaml(repository / "profiles" / "con" / "profile.yaml") + if profile is None + else profile + ) + verify_successor_history( + profile, + repository, + require_terminal=True, + ) + require_clean_checkout(repository, "full-migration website") + require_no_ignored_files(repository, "full-migration website") + require_clean_checkout(ROOT, "full-migration coordinator") + require_no_ignored_files( + UPSTREAM, + "www-from-model hydration transport", + ("assets", "static", "themes/congo"), + ) + + +def verify_declared_pins(profile: dict[str, Any]) -> None: + """Require the profile provenance to match the checked-out gitlinks.""" + schema = profile.get("schema", {}) + components = profile.get("components", {}) + if not isinstance(schema, dict) or not isinstance(components, dict): + raise ProjectionError("Profile schema/components must be mappings") + exact = { + "schema.commit": ( + schema.get("commit"), + ROOT / "submodules" / "things-schemas", + ), + "components.dump_things.commit": ( + components.get("dump_things", {}).get("commit"), + ROOT / "submodules" / "dump-things-service", + ), + "components.dump_things_client.commit": ( + components.get("dump_things_client", {}).get("commit"), + ROOT / "submodules" / "dump-things-pyclient", + ), + "components.qri.commit": ( + components.get("qri", {}).get("commit"), + ROOT / "submodules" / "query-things", + ), + "components.graph.commit": ( + components.get("graph", {}).get("commit"), + ROOT / "submodules" / "things-graph-renderer", + ), + } + for label, (declared, repository) in exact.items(): + actual = git_commit(repository) + if declared != actual: + raise ProjectionError( + f"{label} declares {declared!r}, but the gitlink is {actual}" + ) + require_clean_checkout(repository, label.removesuffix(".commit")) + + declared_congo = components.get("congo", {}).get("commit") + actual_congo = git_tree_object(SITE, "HEAD:themes/congo") + if declared_congo != actual_congo: + raise ProjectionError( + "components.congo.commit declares " + f"{declared_congo!r}, but the site gitlink is {actual_congo}" + ) + + verify_successor_history(profile) + verify_site_worktree_isolation() + require_clean_checkout(UPSTREAM, "www-from-model hydration transport") + verify_transport_trees() + build = profile.get("build", {}) + if not isinstance(build, dict) or build.get("metadata_collection") != COLLECTION: + raise ProjectionError( + f"Profile build.metadata_collection must be {COLLECTION!r}" + ) + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.is_file(): + raise ProjectionError(f"Required YAML file is absent: {path}") + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ProjectionError(f"Expected a YAML mapping: {path}") + return value + + +def site_manifest_path(value: Any, label: str) -> Path: + """Resolve one manifest path while keeping it inside the site checkout.""" + if not isinstance(value, str) or not value or Path(value).is_absolute(): + raise ProjectionError(f"{label} must be a relative site path") + path = (SITE / value).resolve() + if path != SITE and SITE not in path.parents: + raise ProjectionError(f"{label} escapes the site checkout: {value}") + return path + + +def require_manifest_path(value: Any, label: str, expected: Path) -> Path: + """Require a site-relative declaration to match one runtime path.""" + try: + expected_relative = expected.relative_to(SITE).as_posix() + except ValueError as error: + raise ProjectionError( + f"Runtime path for {label} is outside the site checkout: {expected}" + ) from error + if value != expected_relative: + raise ProjectionError( + f"{label} declares {value!r}, but the runtime uses {expected_relative!r}" + ) + path = site_manifest_path(value, label) + if path != expected.resolve(): + raise ProjectionError( + f"{label} declares {path}, but the runtime uses {expected.resolve()}" + ) + return path + + +def unique_strings(value: Any, label: str) -> tuple[str, ...]: + """Load a non-empty manifest string sequence without silent duplicates.""" + if ( + not isinstance(value, list) + or not value + or not all(isinstance(item, str) and item for item in value) + ): + raise ProjectionError(f"{label} must be a non-empty string list") + if len(set(value)) != len(value): + raise ProjectionError(f"{label} contains duplicate values") + return tuple(value) + + +def producer_mapping(path: Path, variable: str) -> dict[str, str]: + """Read a literal producer mapping without importing upstream code.""" + if not path.is_file(): + raise ProjectionError(f"Graph producer is absent: {path}") + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except SyntaxError as error: + raise ProjectionError( + f"Cannot inspect graph producer {path}: {error}" + ) from error + for statement in tree.body: + if not isinstance(statement, (ast.Assign, ast.AnnAssign)): + continue + targets = ( + statement.targets + if isinstance(statement, ast.Assign) + else [statement.target] + ) + if not any( + isinstance(target, ast.Name) and target.id == variable for target in targets + ): + continue + try: + value = ast.literal_eval(statement.value) + except (ValueError, TypeError) as error: + raise ProjectionError( + f"Graph producer {variable} must remain a literal mapping" + ) from error + if not isinstance(value, dict) or not all( + isinstance(key, str) and isinstance(item, str) + for key, item in value.items() + ): + raise ProjectionError(f"Graph producer {variable} is not a string mapping") + return value + raise ProjectionError(f"Graph producer does not declare {variable}") + + +def load_projection_contract( + profile: dict[str, Any] | None = None, + specification: dict[str, Any] | None = None, +) -> ProjectionContract: + """Load and cross-check the executable CON profile manifests.""" + profile = load_yaml(PROFILE_PATH) if profile is None else profile + specification = ( + load_yaml(PROJECTION_SPEC_PATH) if specification is None else specification + ) + + paths = profile.get("paths") + inputs = specification.get("inputs") + if not isinstance(paths, dict) or not isinstance(inputs, dict): + raise ProjectionError("Profile paths and projection inputs must be mappings") + canonical_root = site_manifest_path( + paths.get("canonical_records"), "profile.paths.canonical_records" + ) + reference_root = site_manifest_path( + paths.get("reference_records"), "profile.paths.reference_records" + ) + for label, profile_path, input_value in ( + ("canonical_records", canonical_root, inputs.get("canonical_records")), + ("reference_records", reference_root, inputs.get("reference_records")), + ): + input_path = site_manifest_path(input_value, f"projection.inputs.{label}") + if input_path != profile_path: + raise ProjectionError( + f"Profile and projection {label} paths disagree: " + f"{profile_path} != {input_path}" + ) + + snapshot_path = require_manifest_path( + paths.get("qri_snapshot"), + "profile.paths.qri_snapshot", + COMMITTED / "records.jsonl", + ) + content_root = require_manifest_path( + paths.get("content"), + "profile.paths.content", + COMMITTED / "content", + ) + graph_output = require_manifest_path( + paths.get("graph"), + "profile.paths.graph", + COMMITTED / "static" / "graph.json", + ) + digest_output = require_manifest_path( + paths.get("digest"), + "profile.paths.digest", + COMMITTED / "SHA256SUMS", + ) + + schema = profile.get("schema") + if not isinstance(schema, dict): + raise ProjectionError("profile.schema must be a mapping") + schema_relative = schema.get("path") + if not isinstance(schema_relative, str) or Path(schema_relative).is_absolute(): + raise ProjectionError("profile.schema.path must be repository-relative") + schema_repository = (ROOT / "submodules" / "things-schemas").resolve() + expected_schema_relative = SCHEMA.relative_to(schema_repository).as_posix() + declared_schema = (schema_repository / schema_relative).resolve() + if ( + schema_relative != expected_schema_relative + or Path(schema_relative).as_posix() != schema_relative + or declared_schema != SCHEMA.resolve() + or schema_repository not in declared_schema.parents + ): + raise ProjectionError( + "profile.schema.path does not select the pinned source schema: " + f"{schema_relative!r}" + ) + + identity = profile.get("identity") + profile_homepage = profile.get("homepage") + render = specification.get("render") + if not all( + isinstance(value, dict) for value in (identity, profile_homepage, render) + ): + raise ProjectionError( + "Profile identity/homepage and projection render must be mappings" + ) + assert isinstance(identity, dict) + assert isinstance(profile_homepage, dict) + assert isinstance(render, dict) + render_homepage = render.get("homepage") + if not isinstance(render_homepage, dict): + raise ProjectionError("projection.render.homepage must be a mapping") + homepage_pids = { + identity.get("homepage_pid"), + profile_homepage.get("pid"), + render_homepage.get("pid"), + } + if len(homepage_pids) != 1 or not all( + isinstance(pid, str) and pid for pid in homepage_pids + ): + raise ProjectionError( + "Profile identity and render manifests disagree on homepage PID" + ) + homepage_pid = next(iter(homepage_pids)) + homepage_class = profile_homepage.get("class") + if not isinstance(homepage_class, str) or not homepage_class: + raise ProjectionError("profile.homepage.class must be a CURIE string") + homepage_record = site_manifest_path( + profile_homepage.get("record"), "profile.homepage.record" + ) + homepage_template = site_manifest_path( + render_homepage.get("template"), "projection.render.homepage.template" + ) + + if render.get("engine") != "qri": + raise ProjectionError("projection.render.engine must be 'qri'") + require_manifest_path( + render.get("content_root"), + "projection.render.content_root", + content_root, + ) + pages = render.get("pages") + if not isinstance(pages, dict) or not pages: + raise ProjectionError("projection.render.pages must be a non-empty mapping") + if not all( + isinstance(class_name, str) + and class_name + and isinstance(template, str) + and template + for class_name, template in pages.items() + ): + raise ProjectionError("projection.render.pages must map classes to paths") + page_templates = { + class_name: site_manifest_path( + template, f"projection.render.pages.{class_name}" + ) + for class_name, template in pages.items() + } + unrendered_classes = frozenset( + unique_strings( + render.get("unrendered_classes"), + "projection.render.unrendered_classes", + ) + ) + overlap = set(page_templates) & unrendered_classes + if overlap: + raise ProjectionError( + f"Rendered and unrendered class declarations overlap: {sorted(overlap)}" + ) + if homepage_class not in page_templates: + raise ProjectionError( + "The homepage class must also declare its ordinary page template" + ) + + graph = specification.get("graph") + if not isinstance(graph, dict): + raise ProjectionError("projection.graph must be a mapping") + if graph.get("missing_external_targets") != "reject": + raise ProjectionError( + "projection.graph.missing_external_targets must be 'reject'" + ) + graph_node_classes = frozenset( + unique_strings(graph.get("node_classes"), "projection.graph.node_classes") + ) + graph_relationship_fields = unique_strings( + graph.get("relationship_fields"), + "projection.graph.relationship_fields", + ) + producer = site_manifest_path(graph.get("producer"), "projection.graph.producer") + require_manifest_path(graph.get("output"), "projection.graph.output", graph_output) + producer_node_classes = set(producer_mapping(producer, "wanted_node_types")) + producer_relationship_fields = set(producer_mapping(producer, "wanted_edge_types")) + if graph_node_classes != producer_node_classes: + raise ProjectionError( + "Declared graph node classes differ from the pinned producer: " + f"declared={sorted(graph_node_classes)}, " + f"producer={sorted(producer_node_classes)}" + ) + if set(graph_relationship_fields) != producer_relationship_fields: + raise ProjectionError( + "Declared graph relationship fields differ from the pinned producer: " + f"declared={sorted(graph_relationship_fields)}, " + f"producer={sorted(producer_relationship_fields)}" + ) + + build = profile.get("build") + snapshot = specification.get("snapshot") + if not isinstance(build, dict) or not isinstance(snapshot, dict): + raise ProjectionError("Profile build and projection snapshot must be mappings") + collection = build.get("metadata_collection") + if ( + not isinstance(collection, str) + or not collection + or snapshot.get("collection") != collection + ): + raise ProjectionError( + "Profile and projection manifests disagree on metadata collection" + ) + if collection != COLLECTION: + raise ProjectionError( + f"Projection runtime supports only collection {COLLECTION!r}, " + f"found {collection!r}" + ) + require_manifest_path( + snapshot.get("records"), + "projection.snapshot.records", + snapshot_path, + ) + if snapshot.get("format") != "qri-record-jsonl": + raise ProjectionError("projection.snapshot.format must be 'qri-record-jsonl'") + if snapshot.get("sort_key") != ["schema_type", "pid"]: + raise ProjectionError("projection.snapshot.sort_key must be [schema_type, pid]") + declared_counts = snapshot.get("expected_records", {}) + if not isinstance(declared_counts, dict) or not all( + category in {"canonical", "reference"} and isinstance(count, int) and count >= 0 + for category, count in declared_counts.items() + ): + raise ProjectionError( + "projection.snapshot.expected_records must contain non-negative " + "canonical/reference counts" + ) + + digest = specification.get("digest") + if not isinstance(digest, dict): + raise ProjectionError("projection.digest must be a mapping") + if digest.get("algorithm") != "sha256": + raise ProjectionError("projection.digest.algorithm must be 'sha256'") + require_manifest_path( + digest.get("output"), + "projection.digest.output", + digest_output, + ) + + for label, path in { + "profile.homepage.record": homepage_record, + "projection.render.homepage.template": homepage_template, + **{ + f"projection.render.pages.{class_name}": template + for class_name, template in page_templates.items() + }, + }.items(): + if not path.is_file(): + raise ProjectionError(f"{label} is absent: {path}") + + return ProjectionContract( + canonical_root=canonical_root, + reference_root=reference_root, + collection=collection, + homepage_pid=homepage_pid, + homepage_class=homepage_class, + homepage_record=homepage_record, + homepage_template=homepage_template, + page_templates=page_templates, + unrendered_classes=unrendered_classes, + graph_node_classes=graph_node_classes, + graph_relationship_fields=graph_relationship_fields, + snapshot_path=snapshot_path, + content_root=content_root, + graph_output=graph_output, + digest_output=digest_output, + ) + + +def safe_reset(path: Path) -> None: + """Replace one named build directory and nothing outside build state.""" + resolved = path.resolve() + build = (ROOT / "build").resolve() + if build not in resolved.parents or resolved == build: + raise ProjectionError(f"Refusing to replace non-build path: {resolved}") + if resolved.exists(): + shutil.rmtree(resolved) + resolved.mkdir(parents=True) + + +def require_contained_input(path: Path, root: Path, label: str) -> Path: + """Reject input symlinks that resolve outside their declared root.""" + resolved_root = root.resolve() + resolved = path.resolve() + if resolved != resolved_root and resolved_root not in resolved.parents: + raise ProjectionError( + f"{label} resolves outside {resolved_root}: {path} -> {resolved}" + ) + return resolved + + +def source_records(root: Path, category: str) -> list[SourceRecord]: + resolved_root = root.resolve() + for candidate in root.rglob("*"): + if candidate.is_symlink(): + require_contained_input( + candidate, + resolved_root, + f"{category} source input", + ) + records: list[SourceRecord] = [] + for path in sorted(root.rglob("*.yaml")): + if path.name == ".dumpthings.yaml": + continue + require_contained_input(path, resolved_root, f"{category} source record") + relative = path.relative_to(root) + if len(relative.parts) < 2: + raise ProjectionError( + f"Record is not stored below a class directory: {path}" + ) + class_name = relative.parts[0] + record = load_yaml(path) + if not isinstance(record.get("pid"), str): + raise ProjectionError(f"Record has no string PID: {path}") + expected_type = f"xyzri:{class_name}" + if record.get("schema_type") != expected_type: + raise ProjectionError( + f"{path}: expected top-level schema_type {expected_type!r}" + ) + records.append(SourceRecord(class_name, record, path, category)) + if not records: + raise ProjectionError(f"No records found below {root}") + return records + + +def source_closure(contract: ProjectionContract) -> list[SourceRecord]: + """Load canonical and reference inventories from their declared roots.""" + return [ + *source_records(contract.canonical_root, "canonical"), + *source_records(contract.reference_root, "reference"), + ] + + +def nested_schema_types(value: Any) -> list[str]: + result: list[str] = [] + if isinstance(value, dict): + schema_type = value.get("schema_type") + if isinstance(schema_type, str): + result.append(schema_type) + for child in value.values(): + result.extend(nested_schema_types(child)) + elif isinstance(value, list): + for child in value: + result.extend(nested_schema_types(child)) + return result + + +def normalized_payload(value: Any) -> str: + """Return one deterministic, complete JSON representation.""" + try: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + except (TypeError, ValueError) as error: + raise ProjectionError( + f"Record payload is not normalized JSON: {error}" + ) from error + + +def record_payload_index( + records: Sequence[dict[str, Any]], +) -> dict[str, str]: + """Index complete normalized payloads while rejecting duplicate PIDs.""" + result: dict[str, str] = {} + for record in records: + pid = record.get("pid") + if not isinstance(pid, str) or not pid: + raise ProjectionError("Record payload has no string PID") + if pid in result: + raise ProjectionError(f"Record payload PID is duplicated: {pid}") + result[pid] = normalized_payload(record) + return result + + +def native_value_fingerprint(value: Any) -> Counter[tuple[str, str]]: + """Capture every qualifier on every nested native Things object.""" + result: Counter[tuple[str, str]] = Counter() + if isinstance(value, dict): + schema_type = value.get("schema_type") + if isinstance(schema_type, str) and schema_type.startswith("dlthings:"): + result[(schema_type, normalized_payload(value))] += 1 + for child in value.values(): + result.update(native_value_fingerprint(child)) + elif isinstance(value, list): + for child in value: + result.update(native_value_fingerprint(child)) + return result + + +def accepted_schema_types(schema: Path) -> set[str]: + view = SchemaView(str(schema)) + return {str(view.get_uri(name, expand=False)) for name in view.all_classes()} + + +def relationship_targets(record: dict[str, Any], field: str) -> Iterator[str]: + """Yield one producer relationship's object PIDs, rejecting bad shapes.""" + values = record.get(field, []) + if values is None: + return + if not isinstance(values, list): + values = [values] + for value in values: + target = value.get("object") if isinstance(value, dict) else value + targets = target if isinstance(target, list) else [target] + if not targets or not all(isinstance(item, str) and item for item in targets): + raise ProjectionError( + f"{record.get('pid', '')}: malformed {field} target" + ) + yield from targets + + +def linked_values( + record: dict[str, Any], relationship_fields: Sequence[str] +) -> Iterator[tuple[str, str]]: + """Yield every record PID/reference reached through native link slots.""" + for field in relationship_fields: + values = record.get(field, []) + if not isinstance(values, list): + values = [values] + for value in values: + for target in relationship_targets({field: value}, field): + yield field, target + if isinstance(value, dict): + roles = value.get("roles", []) + if not isinstance(roles, list): + roles = [roles] + for role in roles: + if not isinstance(role, str) or not role: + raise ProjectionError( + f"{record.get('pid', '')}: malformed " + f"{field}.roles target" + ) + yield f"{field}.roles", role + for field in ("kind", "rules"): + values = record.get(field, []) + if not isinstance(values, list): + values = [values] + for value in values: + target = value.get("object") if isinstance(value, dict) else value + if not isinstance(target, str) or not target: + raise ProjectionError( + f"{record.get('pid', '')}: malformed {field} target" + ) + yield field, target + identifiers = record.get("identifiers", []) + if not isinstance(identifiers, list): + identifiers = [identifiers] + for identifier in identifiers: + if not isinstance(identifier, dict) or "creator" not in identifier: + continue + creator = identifier["creator"] + if not isinstance(creator, str) or not creator: + raise ProjectionError( + f"{record.get('pid', '')}: malformed " + "identifiers.creator target" + ) + yield "identifiers.creator", creator + + +def entity_route(pid: str) -> str: + """Return the upstream qri/Hugo route encoded by one web record PID.""" + prefix = "xyzrins:" + if not pid.startswith(prefix): + raise ProjectionError( + f"Renderable record PID does not use the xyzrins namespace: {pid}" + ) + route = pid.removeprefix(prefix).strip("/") + if ( + not route + or route == "." + or any(part in {"", ".", ".."} for part in route.split("/")) + ): + raise ProjectionError(f"Renderable record has an unsafe route: {pid}") + return route + + +def validate_record_contract( + records: list[SourceRecord], + contract: ProjectionContract | None = None, +) -> ProjectionExpectations: + """Validate source closure and derive its pages and native graph.""" + contract = load_projection_contract() if contract is None else contract + unexpected = [ + record.path + for record in records + if record.category not in {"canonical", "reference"} + ] + if unexpected: + raise ProjectionError(f"Unexpected non-source records: {unexpected}") + + canonical = { + record.record["pid"] for record in records if record.category == "canonical" + } + references = { + record.record["pid"] for record in records if record.category == "reference" + } + if not canonical: + raise ProjectionError("The canonical record inventory is empty") + if not references: + raise ProjectionError("The reference record inventory is empty") + + by_pid = {record.record["pid"]: record for record in records} + if len(by_pid) != len(records): + raise ProjectionError("Record PIDs must be unique") + homepage = by_pid.get(contract.homepage_pid) + if homepage is None or homepage.category != "canonical": + raise ProjectionError( + f"Homepage PID is not a canonical record: {contract.homepage_pid}" + ) + if homepage.record.get("schema_type") != contract.homepage_class: + raise ProjectionError( + f"Homepage {contract.homepage_pid} must be " + f"{contract.homepage_class}, found " + f"{homepage.record.get('schema_type')}" + ) + if homepage.path.resolve() != contract.homepage_record: + raise ProjectionError( + "The declared homepage record path does not contain the homepage PID" + ) + + accepted = accepted_schema_types(SCHEMA) + adjacency: dict[str, set[str]] = defaultdict(set) + for item in records: + record = item.record + for schema_type in nested_schema_types(record): + if schema_type.startswith(("http://", "https://")): + raise ProjectionError( + f"{record['pid']}: full-URI type designator is unsupported: " + f"{schema_type}" + ) + if schema_type not in accepted: + raise ProjectionError( + f"{record['pid']}: unknown CURIE type designator: {schema_type}" + ) + + for attribute in record.get("attributes", []): + if not isinstance(attribute, dict): + continue + if attribute.get("predicate") in FORBIDDEN_BRIDGE_PREDICATES: + raise ProjectionError( + f"{record['pid']}: AttributeSpecification cannot encode " + f"relationship predicate {attribute.get('predicate')}" + ) + for field, target in linked_values(record, contract.graph_relationship_fields): + if target not in by_pid: + raise ProjectionError( + f"{record['pid']}: dangling {field} target {target}" + ) + adjacency[record["pid"]].add(target) + + reachable = set(canonical) + pending = list(canonical) + while pending: + source = pending.pop() + for target in adjacency[source]: + if target not in reachable: + reachable.add(target) + pending.append(target) + unused_references = references - reachable + if unused_references: + raise ProjectionError( + "Reference records are outside the canonical native-link closure: " + f"{sorted(unused_references)}" + ) + + graph_node_pids = { + item.record["pid"] + for item in records + if item.category == "canonical" + and item.record["schema_type"] in contract.graph_node_classes + } + graph_reference_classes = { + item.record["schema_type"] + for item in records + if item.category == "reference" + and item.record["schema_type"] in contract.graph_node_classes + } + if graph_reference_classes: + raise ProjectionError( + "Reference classes would materialize as graph nodes: " + f"{sorted(graph_reference_classes)}" + ) + graph_edges: set[tuple[str, str]] = set() + for item in records: + pid = item.record["pid"] + if pid not in graph_node_pids: + continue + for field in contract.graph_relationship_fields: + for target in relationship_targets(item.record, field): + if target not in graph_node_pids: + raise ProjectionError( + f"{pid}: native graph target {target} from {field} " + "does not materialize as a canonical graph node" + ) + graph_edges.add((pid, target)) + + declared_classes = set(contract.page_templates) | set(contract.unrendered_classes) + record_classes = {item.record["schema_type"] for item in records} + undeclared_classes = record_classes - declared_classes + if undeclared_classes: + raise ProjectionError( + "Record classes have no rendered/unrendered policy: " + f"{sorted(undeclared_classes)}" + ) + rendered_references = { + item.record["schema_type"] + for item in records + if item.category == "reference" + and item.record["schema_type"] in contract.page_templates + } + if rendered_references: + raise ProjectionError( + "Reference classes cannot produce entity pages: " + f"{sorted(rendered_references)}" + ) + + entity_routes = { + entity_route(item.record["pid"]) + for item in records + if item.category == "canonical" + and item.record["pid"] != contract.homepage_pid + and item.record["schema_type"] in contract.page_templates + } + markdown_pages = { + "_index.md", + *(f"{route}/_index.md" for route in entity_routes), + } + return ProjectionExpectations( + canonical_pids=frozenset(canonical), + reference_pids=frozenset(references), + graph_node_pids=frozenset(graph_node_pids), + graph_edges=frozenset(graph_edges), + markdown_pages=frozenset(markdown_pages), + entity_routes=frozenset(entity_routes), + record_payloads=tuple( + sorted( + (pid, normalized_payload(item.record)) for pid, item in by_pid.items() + ) + ), + ) + + +def roundtrip_records(records: list[SourceRecord]) -> None: + """Exercise every record through the pinned JSON/RDF conversion path.""" + to_ttl = FormatConverter(str(SCHEMA), Format.json, Format.ttl) + to_json = FormatConverter(str(SCHEMA), Format.ttl, Format.json) + for item in records: + before = Counter(nested_schema_types(item.record)) + before_values = native_value_fingerprint(item.record) + try: + ttl = to_ttl.convert(item.record, item.class_name) + restored = to_json.convert(ttl, item.class_name) + except Exception as error: + raise ProjectionError( + f"{item.record['pid']}: JSON/RDF/JSON round trip failed: {error}" + ) from error + after = Counter(nested_schema_types(restored)) + for schema_type, count in before.items(): + if after[schema_type] < count: + raise ProjectionError( + f"{item.record['pid']}: round trip lost {schema_type}" + ) + after_values = native_value_fingerprint(restored) + if after_values != before_values: + raise ProjectionError( + f"{item.record['pid']}: round trip changed native-object qualifiers" + ) + + association = next( + item + for item in records + if "dlthings:Association" in nested_schema_types(item.record) + ) + invalid = deepcopy(association.record) + + def expand_first(value: Any) -> bool: + if isinstance(value, dict): + if value.get("schema_type") == "dlthings:Association": + value["schema_type"] = ( + "https://concepts.datalad.org/s/things/v2/Association" + ) + return True + return any(expand_first(child) for child in value.values()) + if isinstance(value, list): + return any(expand_first(child) for child in value) + return False + + if not expand_first(invalid): + raise ProjectionError("No Association fixture was available") + try: + to_ttl.convert(invalid, association.class_name) + except Exception: + pass + else: + raise ProjectionError( + "Pinned conversion unexpectedly accepted a full-URI discriminator" + ) + + +def write_record_store(records: list[SourceRecord], root: Path) -> Path: + curated = root / COLLECTION / "curated" + incoming = root / COLLECTION / "incoming" + curated.mkdir(parents=True) + incoming.mkdir(parents=True) + (curated / ".dumpthings.yaml").write_text( + yaml.safe_dump( + { + "type": "records", + "version": 1, + "schema": str(SCHEMA.resolve()), + "format": "yaml", + "idfx": "after-last-colon", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + by_class: dict[str, list[SourceRecord]] = defaultdict(list) + for record in records: + by_class[record.class_name].append(record) + for class_name, items in sorted(by_class.items()): + destination = curated / class_name + destination.mkdir() + for index, item in enumerate( + sorted(items, key=lambda value: value.record["pid"]), start=1 + ): + output = destination / f"{index:02d}.yaml" + output.write_text( + yaml.safe_dump( + item.record, + sort_keys=False, + allow_unicode=True, + ), + encoding="utf-8", + ) + return root + + +def write_service_config(records: list[SourceRecord], root: Path) -> Path: + # qri's upstream inject-links command queries the polymorphic Thing + # endpoint, in addition to the concrete classes present in the slice. + classes = sorted({"Thing", *(record.class_name for record in records)}) + config = { + "type": "collections", + "version": 2, + "pid": "dump_things:clean_migration_projection", + "collections": { + COLLECTION: { + "default_token": READER_TOKEN, + "schema": str(SCHEMA.resolve()), + "curated": f"{COLLECTION}/curated", + "incoming": f"{COLLECTION}/incoming", + "backend": { + "type": "record_dir+stl", + "mapping_method": "after-last-colon", + }, + "auth_sources": [{"type": "config"}], + "use_classes": classes, + } + }, + "tokens": { + READER_TOKEN: { + "user_id": READER_TOKEN, + "representation": READER_TOKEN, + "collections": { + COLLECTION: { + "mode": "READ_CURATED", + "incoming_label": "", + } + }, + }, + VALIDATOR_TOKEN: { + "user_id": VALIDATOR_TOKEN, + "representation": VALIDATOR_TOKEN, + "collections": { + COLLECTION: { + "mode": "WRITE_COLLECTION", + "incoming_label": "validation", + } + }, + }, + }, + "admin_tokens": {}, + } + path = root / "config.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + return path + + +def free_port() -> int: + with socket.socket() as candidate: + candidate.bind(("127.0.0.1", 0)) + return int(candidate.getsockname()[1]) + + +def wait_for_service(url: str, process: subprocess.Popen[str]) -> None: + for _ in range(480): + if process.poll() is not None: + raise ProjectionError( + f"Ephemeral Dump Things exited with status {process.returncode}" + ) + try: + with urlopen(f"{url}/server", timeout=1): + return + except (URLError, TimeoutError): + time.sleep(0.25) + raise ProjectionError("Ephemeral Dump Things did not become ready") + + +@contextmanager +def dump_things_service(records: list[SourceRecord], state: Path) -> Iterator[str]: + store = write_record_store(records, state / "store") + config = write_service_config(records, store) + port = free_port() + url = f"http://127.0.0.1:{port}" + log_path = state / "dump-things.log" + environment = os.environ.copy() + environment["DTS_ADMIN_TOKEN"] = "clean-migration-local-admin" + with log_path.open("w", encoding="utf-8") as log: + process = subprocess.Popen( + [ + "dump-things-service", + str(store), + "--config", + str(config), + "--host", + "127.0.0.1", + "--port", + str(port), + "--log-level", + "WARNING", + ], + cwd=ROOT, + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + try: + wait_for_service(url, process) + yield url + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def request_validation( + url: str, + item: SourceRecord, + *, + expected_status: int = 200, +) -> None: + request = Request( + f"{url}/{COLLECTION}/validate/record/{item.class_name}", + data=json.dumps(item.record).encode("utf-8"), + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "X-DumpThings-Token": VALIDATOR_TOKEN, + }, + ) + try: + with urlopen(request, timeout=120) as response: + status = response.status + except HTTPError as error: + status = error.code + if status != expected_status: + raise ProjectionError( + f"Live validation for {item.record['pid']} returned {status}, " + f"expected {expected_status}" + ) + + +def live_negative_cases(url: str, records: list[SourceRecord]) -> None: + association = next( + item + for item in records + if "dlthings:Association" in nested_schema_types(item.record) + ) + full_uri = deepcopy(association.record) + unknown = deepcopy(association.record) + + def replace(value: Any, replacement: str) -> bool: + if isinstance(value, dict): + if value.get("schema_type") == "dlthings:Association": + value["schema_type"] = replacement + return True + return any(replace(child, replacement) for child in value.values()) + if isinstance(value, list): + return any(replace(child, replacement) for child in value) + return False + + replace( + full_uri, + "https://concepts.datalad.org/s/things/v2/Association", + ) + replace(unknown, "dlthings:NotARealAssociation") + for record in (full_uri, unknown): + request_validation( + url, + SourceRecord( + association.class_name, + record, + association.path, + "negative", + ), + expected_status=422, + ) + + +def service_export( + url: str, records: list[SourceRecord], state: Path +) -> list[dict[str, Any]]: + by_class: dict[str, list[SourceRecord]] = defaultdict(list) + for record in records: + by_class[record.class_name].append(record) + request_validation(url, record) + live_negative_cases(url, records) + + environment = os.environ.copy() + environment["DTC_TOKEN"] = VALIDATOR_TOKEN + validation_log: list[str] = [] + for class_name, items in sorted(by_class.items()): + stream = "".join( + json.dumps(item.record, sort_keys=True) + "\n" + for item in sorted(items, key=lambda value: value.record["pid"]) + ) + validation_log.append( + run( + ["dtc", "post-records", url, COLLECTION, class_name], + input_text=stream, + environment=environment, + action=f"Validate {class_name} records through dtc", + ) + ) + (state / "dtc-validation.log").write_text("".join(validation_log), encoding="utf-8") + + environment["DTC_TOKEN"] = READER_TOKEN + exported = run( + ["dtc", "get-records", url, COLLECTION], + environment=environment, + action="Export validated CON records through dtc", + ) + parsed = [json.loads(line) for line in exported.splitlines() if line.strip()] + parsed.sort(key=lambda record: (record["schema_type"], record["pid"])) + expected = record_payload_index([item.record for item in records]) + actual = record_payload_index(parsed) + if actual != expected: + mismatched = sorted( + pid + for pid in set(actual) | set(expected) + if actual.get(pid) != expected.get(pid) + ) + raise ProjectionError( + "dtc export payload differs from the normalized source closure: " + f"{mismatched}" + ) + return parsed + + +def qri_pipeline( + commands: list[list[str | Path]], + environment: dict[str, str], + *, + input_text: str | None = None, + action: str, +) -> str: + output = input_text + for command in commands: + output = run( + command, + input_text=output, + environment=environment, + action=action, + ) + return output or "" + + +def upstream_qri_pipelines( + contract: ProjectionContract, + workflow_path: Path | None = None, +) -> tuple[dict[str, list[list[str]]], list[list[str]]]: + """Derive page-selection pipelines from the pinned upstream workflow.""" + path = SITE / UPSTREAM_UPDATE_WORKFLOW if workflow_path is None else workflow_path + workflow = load_yaml(path) + jobs = workflow.get("jobs") + if not isinstance(jobs, dict): + raise ProjectionError("Pinned upstream workflow has no jobs mapping") + create_pages = jobs.get("create_pages") + if not isinstance(create_pages, dict): + raise ProjectionError("Pinned upstream workflow has no create_pages job") + steps = create_pages.get("steps") + if not isinstance(steps, list): + raise ProjectionError("Pinned upstream create_pages job has no steps") + + page_pipelines: dict[str, list[list[str]]] = {} + page_templates: dict[str, Path] = {} + homepage_pipeline: list[list[str]] | None = None + homepage_template: Path | None = None + expected_output = "content/{__pid_curie_reference}/_index.md" + for step in steps: + if not isinstance(step, dict) or not isinstance(step.get("run"), str): + continue + script = re.sub(r"\\\s*\n\s*", " ", step["run"]).strip() + segments = [segment.strip() for segment in script.split("|")] + if not segments or not segments[0].startswith("qri list "): + continue + try: + commands = [shlex.split(segment) for segment in segments] + except ValueError as error: + raise ProjectionError( + f"Cannot parse pinned upstream qri pipeline: {error}" + ) from error + if any(not command or command[0] != "qri" for command in commands): + raise ProjectionError( + "Pinned upstream page pipeline contains a non-qri command" + ) + renderer = commands[-1] + if len(renderer) != 4 or renderer[:2] != ["qri", "render-record"]: + raise ProjectionError( + "Pinned upstream page pipeline has an unexpected renderer" + ) + if renderer[3] != expected_output: + raise ProjectionError( + "Pinned upstream page pipeline has an unexpected output path: " + f"{renderer[3]!r}" + ) + template = site_manifest_path( + renderer[2], "pinned upstream qri render template" + ) + selection = commands[:-1] + first = selection[0] + is_homepage = first[:3] == ["qri", "list", "--pid"] + if is_homepage: + if len(first) != 4 or first[3] != "xyzrins:.": + raise ProjectionError( + "Pinned upstream homepage pipeline selects an unexpected PID" + ) + if homepage_pipeline is not None: + raise ProjectionError( + "Pinned upstream workflow defines multiple homepage pipelines" + ) + else: + if len(first) != 4 or first[:3] != ["qri", "list", "--class"]: + raise ProjectionError( + "Pinned upstream page pipeline has an unexpected selector" + ) + schema_type = first[3] + if schema_type in page_pipelines: + raise ProjectionError( + "Pinned upstream workflow defines multiple pipelines for " + f"{schema_type}" + ) + + adapted: list[list[str]] = [] + for command in selection: + command = [ + contract.collection if token == "public" else token for token in command + ] + command = [ + contract.homepage_pid if token == "xyzrins:." else token + for token in command + ] + if command[:2] == ["qri", "inject-links-pid"] and not any( + token in {"-c", "--collection"} for token in command + ): + command.extend(["-c", contract.collection]) + adapted.append(command) + + if is_homepage: + homepage_pipeline = adapted + homepage_template = template + else: + page_pipelines[schema_type] = adapted + page_templates[schema_type] = template + + if homepage_pipeline is None or homepage_template is None: + raise ProjectionError("Pinned upstream workflow has no homepage qri pipeline") + if homepage_template != contract.homepage_template: + raise ProjectionError( + "Declared homepage template differs from the pinned upstream pipeline" + ) + unsupported = set(contract.page_templates) - set(page_pipelines) + if unsupported: + raise ProjectionError( + f"Rendered classes have no pinned qri pipeline: {sorted(unsupported)}" + ) + mismatched = { + schema_type: (contract.page_templates[schema_type], page_templates[schema_type]) + for schema_type in contract.page_templates + if contract.page_templates[schema_type] != page_templates[schema_type] + } + if mismatched: + raise ProjectionError( + "Declared page templates differ from pinned upstream pipelines: " + f"{mismatched}" + ) + return page_pipelines, homepage_pipeline + + +def render_qri( + url: str, + records: list[dict[str, Any]], + output: Path, + state: Path, + contract: ProjectionContract, +) -> None: + stream = "".join( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + for record in records + ) + cache = state / "qri-cache.json" + environment = os.environ.copy() + environment.update( + { + "DUMPTHINGS_APIURL": url, + "DUMPTHINGS_TOKEN": READER_TOKEN, + "QRI_RECORD_CACHE": str(cache), + } + ) + qri_pipeline( + [["qri", "cache"]], + environment, + input_text=stream, + action="Cache the dtc export with qri", + ) + + content = output / "content" + selection_commands, homepage_commands = upstream_qri_pipelines(contract) + for schema_type, page_template in sorted(contract.page_templates.items()): + commands = selection_commands[schema_type] + name = schema_type.removeprefix("xyzri:XYZ").lower() + selected = qri_pipeline( + commands, + environment, + action=f"Select and inline the CON {name} projection", + ) + output_template = str(content / "{__pid_curie_reference}" / "_index.md") + qri_pipeline( + [["qri", "render-record", page_template, output_template]], + environment, + input_text=selected, + action=f"Render the CON {name} projection", + ) + + homepage = qri_pipeline( + homepage_commands, + environment, + action="Select and inline the CON homepage projection", + ) + qri_pipeline( + [ + [ + "qri", + "render-record", + contract.homepage_template, + content / "_index.md", + ] + ], + environment, + input_text=homepage, + action="Render the CON homepage projection", + ) + + all_records = run( + ["qri", "list"], + environment=environment, + action="List qri records for the upstream graph", + ) + graph = run( + [sys.executable, SITE / "code" / "pool2graph.py"], + input_text=all_records, + environment=environment, + action="Render the graph with upstream pool2graph.py", + ) + graph_path = output / "static" / "graph.json" + graph_path.parent.mkdir(parents=True) + graph_path.write_text(graph + "\n", encoding="utf-8") + + +def validate_projection( + records: list[dict[str, Any]], + output: Path, + expectations: ProjectionExpectations | None = None, +) -> dict[str, Any]: + if expectations is None: + contract = load_projection_contract() + expectations = validate_record_contract(source_closure(contract), contract) + graph = json.loads((output / "static" / "graph.json").read_text()) + nodes = graph.get("nodes") + edges = graph.get("edges") + if not isinstance(nodes, list) or not isinstance(edges, list): + raise ProjectionError("Upstream graph output has no node/edge lists") + node_pids = {node.get("id") for node in nodes} + if node_pids != expectations.graph_node_pids or len(nodes) != len( + expectations.graph_node_pids + ): + raise ProjectionError( + "CON graph node closure differs: " + f"expected={sorted(expectations.graph_node_pids)}, " + f"actual={sorted(node_pids)}" + ) + edge_pairs = {(edge.get("source"), edge.get("target")) for edge in edges} + if edge_pairs != expectations.graph_edges or len(edges) != len( + expectations.graph_edges + ): + raise ProjectionError( + "CON graph edge closure differs: " + f"expected={sorted(expectations.graph_edges)}, " + f"actual={sorted(edge_pairs)}" + ) + actual_payloads = record_payload_index(records) + expected_payloads = dict(expectations.record_payloads) + if actual_payloads != expected_payloads: + mismatched = sorted( + pid + for pid in set(actual_payloads) | set(expected_payloads) + if actual_payloads.get(pid) != expected_payloads.get(pid) + ) + raise ProjectionError( + f"qri record payload differs from source inventory: {mismatched}" + ) + + actual_pages = { + path.relative_to(output / "content").as_posix() + for path in (output / "content").rglob("*.md") + } + if actual_pages != expectations.markdown_pages: + raise ProjectionError( + "Unexpected qri page closure: " + f"expected={sorted(expectations.markdown_pages)}, " + f"actual={sorted(actual_pages)}" + ) + return { + "records": len(records), + "canonical_records": len(expectations.canonical_pids), + "reference_records": len(expectations.reference_pids), + "generated_records": 0, + "graph_nodes": len(nodes), + "graph_edges": len(edges), + "pages": len(actual_pages), + "native_edges": sorted([list(pair) for pair in edge_pairs]), + } + + +def files_below(root: Path) -> Iterator[Path]: + if not root.exists(): + return + resolved_root = root.resolve() + for path in sorted(root.rglob("*")): + if path.is_symlink(): + require_contained_input(path, resolved_root, "Scoped file input") + if path.is_file() and path.name != ".DS_Store": + yield path + + +def scoped_digest_path(value: str) -> tuple[str, Path, Path]: + """Resolve one digest scope entry to its labeled repository root.""" + if value.startswith("upstream:"): + label = "upstream" + root = SITE + relative = value.removeprefix("upstream:") + elif value.startswith("parent:"): + label = "parent" + root = ROOT + relative = value.removeprefix("parent:") + else: + label = "site" + root = SITE + relative = value + if not relative or Path(relative).is_absolute(): + raise ProjectionError(f"Invalid projection digest scope path: {value}") + path = root / relative + require_contained_input(path, root, f"Projection digest scope {value}") + return label, path, root + + +def input_files( + specification: dict[str, Any] | None = None, +) -> list[tuple[str, Path]]: + """Expand the projection manifest's explicit metadata-only file scope.""" + specification = ( + load_yaml(PROJECTION_SPEC_PATH) if specification is None else specification + ) + digest = specification.get("digest") + if not isinstance(digest, dict): + raise ProjectionError("projection.digest must be a mapping") + scope = unique_strings(digest.get("scope"), "projection.digest.scope") + sentinels = {"component-commit-pins", "projection-runtime-pins"} + missing_sentinels = sentinels - set(scope) + if missing_sentinels: + raise ProjectionError( + "Projection digest scope omits required pin sets: " + f"{sorted(missing_sentinels)}" + ) + + entries: list[tuple[str, Path]] = [] + for item in scope: + if item in sentinels: + continue + label, path, repository_root = scoped_digest_path(item) + resolved_path = path.resolve() + if resolved_path == COMMITTED.resolve() or COMMITTED.resolve() in ( + resolved_path.parents + ): + raise ProjectionError( + f"Projection outputs cannot also be digest inputs: {item}" + ) + if path.is_file(): + paths = [path] + elif path.is_dir(): + paths = list(files_below(path)) + else: + raise ProjectionError(f"Projection digest input is absent: {path}") + for candidate in paths: + require_contained_input( + candidate, + resolved_path if path.is_dir() else repository_root, + f"Projection digest input {item}", + ) + relative = candidate.relative_to(repository_root).as_posix() + entries.append((f"{label}/{relative}", candidate)) + labels = [label for label, _ in entries] + if len(labels) != len(set(labels)): + raise ProjectionError("Projection digest scope names an input twice") + return sorted(entries, key=lambda item: item[0]) + + +def projection_local_runtime_pins(config: dict[str, Any]) -> dict[str, str]: + """Bind local Pixi package paths and overrides to pinned submodules.""" + pypi = config.get("pypi-dependencies", {}) + if not isinstance(pypi, dict): + raise ProjectionError("Pixi PyPI dependency table is missing") + local_pins: dict[str, str] = {} + for name, path in PROJECTION_LOCAL_PYPI_PATHS.items(): + declaration = pypi.get(name) + expected = {"path": path} + if declaration != expected: + raise ProjectionError( + f"Projection runtime {name} must be declared exactly as {expected}, " + f"found {declaration!r}" + ) + local_pins[f"local:{name}"] = f"path={path}" + + options = config.get("pypi-options", {}) + overrides = ( + options.get("dependency-overrides") if isinstance(options, dict) else None + ) + expected_overrides = { + "dump-things-pyclient": { + "path": PROJECTION_LOCAL_PYPI_PATHS["dump-things-pyclient"] + } + } + if overrides != expected_overrides: + raise ProjectionError( + "Projection runtime dependency overrides must be declared exactly as " + f"{expected_overrides}, found {overrides!r}" + ) + local_pins["override:dump-things-pyclient"] = ( + "path=" + PROJECTION_LOCAL_PYPI_PATHS["dump-things-pyclient"] + ) + return local_pins + + +def projection_runtime_pins() -> list[tuple[str, str]]: + """Return only direct runtimes that can alter metadata projection bytes.""" + config = tomllib.loads((ROOT / "pixi.toml").read_text(encoding="utf-8")) + conda = config.get("dependencies", {}) + pypi = config.get("pypi-dependencies", {}) + if not isinstance(conda, dict) or not isinstance(pypi, dict): + raise ProjectionError("Pixi dependency tables are missing") + sources = { + "python": conda.get("python"), + "jinja2": pypi.get("jinja2"), + "packaging": pypi.get("packaging"), + "pyyaml": pypi.get("pyyaml"), + "linkml": pypi.get("linkml"), + "linkml-runtime": pypi.get("linkml-runtime"), + "pydantic": pypi.get("pydantic"), + "rdflib": pypi.get("rdflib"), + } + if not all(isinstance(value, str) and value for value in sources.values()): + raise ProjectionError( + "Projection runtime dependencies must use direct string pins" + ) + local_pins = projection_local_runtime_pins(config) + return sorted( + [*(sources.items()), *(local_pins.items())], + key=lambda item: item[0], + ) + + +def conda_package_name(reference: str, known_names: set[str]) -> str: + """Recover a Conda package name from one resolved artifact reference.""" + filename = Path(urlsplit(reference).path).name + for suffix in (".conda", ".tar.bz2"): + if filename.endswith(suffix): + filename = filename[: -len(suffix)] + break + matches = [name for name in known_names if filename.startswith(f"{name}-")] + if not matches: + raise ProjectionError( + f"Cannot identify resolved Conda package from {reference!r}" + ) + return max(matches, key=len) + + +def conda_package_version(reference: str, name: str) -> str: + """Return the version encoded in one resolved Conda artifact name.""" + filename = Path(urlsplit(reference).path).name + for suffix in (".conda", ".tar.bz2"): + if filename.endswith(suffix): + filename = filename[: -len(suffix)] + break + remainder = filename.removeprefix(f"{name}-") + version, separator, _ = remainder.partition("-") + if not separator or not version: + raise ProjectionError( + f"Cannot identify resolved Conda version from {reference!r}" + ) + return version + + +def lock_platform_environment( + platform: str, + subdir: str, + python_version: str, +) -> dict[str, str]: + """Return deterministic marker values for one locked target platform.""" + major_minor = ".".join(python_version.split(".")[:2]) + environment = { + "implementation_name": "cpython", + "implementation_version": python_version, + "os_name": "posix", + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_version": "", + "python_full_version": python_version, + "python_version": major_minor, + } + if subdir == "linux-64": + environment.update( + { + "platform_machine": "x86_64", + "platform_system": "Linux", + "sys_platform": "linux", + } + ) + elif subdir == "osx-arm64": + environment.update( + { + "platform_machine": "arm64", + "platform_system": "Darwin", + "sys_platform": "darwin", + } + ) + else: + raise ProjectionError( + f"Projection runtime does not define marker values for {platform}: {subdir}" + ) + return environment + + +def require_deterministic_marker(requirement: Requirement) -> None: + """Reject requirement markers that depend on an unspecified host kernel.""" + marker_text = str(requirement.marker or "") + if re.search( + r"\b(?:platform_release|platform_version)\b", + marker_text, + ): + raise ProjectionError( + f"Projection dependency uses a host-specific marker: {str(requirement)!r}" + ) + + +def projection_runtime_lock_records( + lock_path: Path | None = None, +) -> list[tuple[str, str]]: + """Resolve projection-only direct/transitive packages from Pixi's lock.""" + lock_path = ROOT / "pixi.lock" if lock_path is None else lock_path + if not lock_path.is_file() or lock_path.is_symlink(): + raise ProjectionError(f"Resolved Pixi lock is absent or symlinked: {lock_path}") + lock = yaml.safe_load(lock_path.read_text(encoding="utf-8")) + if not isinstance(lock, dict) or lock.get("version") != 7: + raise ProjectionError("Projection runtime requires Pixi lock format 7") + packages = lock.get("packages") + environments = lock.get("environments") + platforms = lock.get("platforms") + if ( + not isinstance(packages, list) + or not isinstance(environments, dict) + or not isinstance(platforms, list) + ): + raise ProjectionError("Pixi lock package/environment tables are malformed") + default = environments.get("default") + if not isinstance(default, dict) or not isinstance(default.get("packages"), dict): + raise ProjectionError("Pixi lock has no default environment package table") + + package_by_reference: dict[tuple[str, str], dict[str, Any]] = {} + conda_names: set[str] = {"python"} + config = tomllib.loads((ROOT / "pixi.toml").read_text(encoding="utf-8")) + direct_conda = config.get("dependencies", {}) + if isinstance(direct_conda, dict): + conda_names.update(str(name) for name in direct_conda) + targets = config.get("target", {}) + if isinstance(targets, dict): + for target in targets.values(): + if isinstance(target, dict) and isinstance( + target.get("dependencies"), dict + ): + conda_names.update(str(name) for name in target["dependencies"]) + for package in packages: + if not isinstance(package, dict): + raise ProjectionError("Pixi lock package entry is not a mapping") + references = [ + (kind, package[kind]) + for kind in ("conda", "pypi") + if isinstance(package.get(kind), str) + ] + if len(references) != 1: + raise ProjectionError( + "Pixi lock package entry must have one Conda or PyPI reference" + ) + kind, reference = references[0] + key = (kind, reference) + if key in package_by_reference: + raise ProjectionError(f"Pixi lock repeats package reference {key}") + package_by_reference[key] = package + for dependency in package.get("depends", []): + if isinstance(dependency, str): + name = dependency.split()[0] + if not name.startswith("__"): + conda_names.add(name) + + platform_subdirs: dict[str, str] = {} + for value in platforms: + if not isinstance(value, dict) or not isinstance(value.get("name"), str): + raise ProjectionError("Pixi lock platform entry is malformed") + name = value["name"] + subdir = value.get("subdir", name) + if not isinstance(subdir, str): + raise ProjectionError(f"Pixi lock platform {name} has no subdir") + platform_subdirs[name] = subdir + + result: list[tuple[str, str]] = [] + for platform, references in sorted(default["packages"].items()): + if platform not in platform_subdirs or not isinstance(references, list): + raise ProjectionError( + f"Pixi lock environment platform is invalid: {platform}" + ) + selected: dict[tuple[str, str], dict[str, Any]] = {} + for reference_item in references: + if not isinstance(reference_item, dict) or len(reference_item) != 1: + raise ProjectionError( + f"Pixi lock reference for {platform} is malformed" + ) + kind, reference = next(iter(reference_item.items())) + if kind not in {"conda", "pypi"} or not isinstance(reference, str): + raise ProjectionError( + f"Pixi lock reference for {platform} is malformed" + ) + package = package_by_reference.get((kind, reference)) + if package is None: + raise ProjectionError( + f"Pixi lock reference has no package entry: {reference}" + ) + if kind == "pypi": + raw_name = package.get("name") + if not isinstance(raw_name, str): + raise ProjectionError(f"PyPI package has no name: {reference}") + name = canonicalize_name(raw_name) + else: + name = conda_package_name(reference, conda_names) + key = (kind, name) + if key in selected: + raise ProjectionError( + f"Pixi lock selects {kind}:{name} twice for {platform}" + ) + selected[key] = package + + python_package = selected.get(("conda", "python")) + if python_package is None: + raise ProjectionError(f"Pixi lock has no Python package for {platform}") + match = re.search( + r"/python-(\d+\.\d+\.\d+)-", + str(python_package["conda"]), + ) + if match is None: + raise ProjectionError( + f"Cannot determine locked Python version for {platform}" + ) + marker_environment = lock_platform_environment( + platform, + platform_subdirs[platform], + match.group(1), + ) + + roots = {("conda", "python"): {""}} + roots.update( + {("pypi", canonicalize_name(name)): {""} for name in PROJECTION_PYPI_ROOTS} + ) + active_extras = {key: set(extras) for key, extras in roots.items()} + pending = list(active_extras) + visited: dict[tuple[str, str], set[str]] = {} + while pending: + key = pending.pop() + extras = active_extras[key] + if visited.get(key) == extras: + continue + visited[key] = set(extras) + package = selected.get(key) + if package is None: + raise ProjectionError( + f"Projection runtime dependency is not locked for {platform}: " + f"{key[0]}:{key[1]}" + ) + dependencies: list[tuple[tuple[str, str], set[str]]] = [] + if key[0] == "conda": + for dependency in package.get("depends", []): + if not isinstance(dependency, str): + raise ProjectionError( + f"Conda dependency for {key[1]} is malformed" + ) + name = dependency.split()[0] + if not name.startswith("__"): + dependencies.append((("conda", name), set())) + else: + for dependency in package.get("requires_dist", []): + if not isinstance(dependency, str): + raise ProjectionError( + f"PyPI dependency for {key[1]} is malformed" + ) + try: + requirement = Requirement(dependency) + except InvalidRequirement as error: + raise ProjectionError( + f"Cannot parse locked requirement {dependency!r}" + ) from error + require_deterministic_marker(requirement) + if requirement.marker is not None and not any( + requirement.marker.evaluate( + {**marker_environment, "extra": extra} + ) + for extra in extras + ): + continue + dependencies.append( + ( + ("pypi", canonicalize_name(requirement.name)), + set(requirement.extras), + ) + ) + for dependency_key, dependency_extras in dependencies: + if dependency_key not in selected: + raise ProjectionError( + f"Projection dependency is unresolved for {platform}: " + f"{key[1]} -> {dependency_key[1]}" + ) + required_extras = {"", *dependency_extras} + if required_extras <= active_extras.get(dependency_key, set()): + continue + active_extras.setdefault(dependency_key, set()).update(required_extras) + pending.append(dependency_key) + + for kind, name in sorted(visited): + package = selected[(kind, name)] + fields = ( + ("conda", "sha256", "md5", "depends", "constrains") + if kind == "conda" + else ( + "pypi", + "name", + "version", + "sha256", + "requires_dist", + "requires_python", + ) + ) + payload = {field: package[field] for field in fields if field in package} + payload["active_extras"] = sorted(visited[(kind, name)]) + payload["platform_subdir"] = platform_subdirs[platform] + version = package.get("version") + if kind == "conda": + identity = conda_package_version(str(package["conda"]), name) + else: + identity = version if isinstance(version, str) else "local" + label = f"{platform_subdirs[platform]}:{kind}:{name}@{identity}" + result.append( + ( + label, + digest_bytes((normalized_payload(payload) + "\n").encode("utf-8")), + ) + ) + return sorted(result) + + +def projection_runtime_lock_digest(lock_path: Path | None = None) -> str: + """Fingerprint only the resolved runtime closure used by projection.""" + records = projection_runtime_lock_records(lock_path) + payload = "".join(f"{digest} {label}\n" for label, digest in records) + return digest_bytes(payload.encode("utf-8")) + + +def projection_component_pins() -> list[tuple[str, str]]: + """Return source/runtime commits that can alter projection bytes.""" + return [ + ( + "things-schemas", + git_commit(ROOT / "submodules" / "things-schemas"), + ), + ( + "dump-things-service", + git_commit(ROOT / "submodules" / "dump-things-service"), + ), + ( + "dump-things-pyclient", + git_commit(ROOT / "submodules" / "dump-things-pyclient"), + ), + ( + "query-things", + git_commit(ROOT / "submodules" / "query-things"), + ), + ] + + +def declared_component_pins() -> list[tuple[str, str]]: + profile = load_yaml(PROFILE_PATH) + components = profile.get("components", {}) + if not isinstance(components, dict): + raise ProjectionError("Profile components must be a mapping") + upstream_base = components.get("www_from_model", {}).get("commit") + if not isinstance(upstream_base, str): + raise ProjectionError("Profile does not pin the upstream website base") + return [ + ("www-from-model", upstream_base), + ( + "things-schemas", + git_commit(ROOT / "submodules" / "things-schemas"), + ), + ( + "dump-things-service", + git_commit(ROOT / "submodules" / "dump-things-service"), + ), + ( + "dump-things-pyclient", + git_commit(ROOT / "submodules" / "dump-things-pyclient"), + ), + ( + "query-things", + git_commit(ROOT / "submodules" / "query-things"), + ), + ( + "things-graph-renderer", + git_commit(ROOT / "submodules" / "things-graph-renderer"), + ), + ( + "congo", + git_tree_object(SITE, "HEAD:themes/congo"), + ), + ] + + +def digest_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def projection_profile_digest_bytes(path: Path) -> bytes: + """Serialize only profile declarations that can alter projection bytes.""" + profile = load_yaml(path) + identity = profile.get("identity") + paths = profile.get("paths") + schema = profile.get("schema") + homepage = profile.get("homepage") + build = profile.get("build") + if not all( + isinstance(value, dict) for value in (identity, paths, schema, homepage, build) + ): + raise ProjectionError("Projection profile contract sections are malformed") + payload = { + "version": profile.get("version"), + "name": profile.get("name"), + "identity": {"homepage_pid": identity.get("homepage_pid")}, + "paths": { + key: paths.get(key) + for key in ( + "canonical_records", + "reference_records", + "qri_snapshot", + "content", + "graph", + "digest", + ) + }, + "schema": {key: schema.get(key) for key in ("path", "discriminator_contract")}, + "homepage": {key: homepage.get(key) for key in ("pid", "class", "record")}, + "build": {"metadata_collection": build.get("metadata_collection")}, + } + return (normalized_payload(payload) + "\n").encode("utf-8") + + +def projection_input_bytes(label: str, path: Path) -> bytes: + """Return the projection-relevant representation of one scoped input.""" + if label == "site/profiles/con/profile.yaml": + return projection_profile_digest_bytes(path) + return path.read_bytes() + + +def projection_manifest(output: Path) -> str: + lines = ["# clean-migration projection manifest v1"] + for label, path in input_files(): + if not path.is_file(): + raise ProjectionError(f"Projection input is absent: {path}") + lines.append( + f"{digest_bytes(projection_input_bytes(label, path))} input:{label}" + ) + for name, commit in projection_component_pins(): + lines.append( + f"{digest_bytes((commit + chr(10)).encode())} pin:{name}@{commit}" + ) + for name, version in projection_runtime_pins(): + value = f"{name}{version}" + lines.append(f"{digest_bytes((value + chr(10)).encode())} pin:runtime:{value}") + for label, digest in projection_runtime_lock_records(): + lines.append(f"{digest} pin:runtime-resolved:{label}") + lines.append( + f"{projection_runtime_lock_digest()} pin:runtime-lock:projection-closure" + ) + for path in files_below(output): + if path.name == "SHA256SUMS" or path.name.startswith("qri-cache"): + continue + relative = path.relative_to(output).as_posix() + lines.append(f"{digest_bytes(path.read_bytes())} output:{relative}") + return "\n".join([lines[0], *sorted(lines[1:])]) + "\n" + + +def verify_manifest(output: Path) -> None: + path = output / "SHA256SUMS" + if not path.is_file(): + raise ProjectionError(f"Committed projection digest is absent: {path}") + expected = projection_manifest(output) + actual = path.read_text(encoding="utf-8") + if actual != expected: + raise ProjectionError( + "The committed CON projection is stale; run " + "`pixi run update-con-projection` after reviewing input changes" + ) + + +def stack_records(records: list[dict[str, Any]], destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text( + "".join( + json.dumps( + { + "class_name": str(record["schema_type"]).rsplit(":", 1)[-1], + "record": record, + }, + ensure_ascii=False, + sort_keys=True, + ) + + "\n" + for record in records + ), + encoding="utf-8", + ) + + +def render_projection(output: Path) -> dict[str, Any]: + if not PROFILE_PATH.is_file() or not PROJECTION_SPEC_PATH.is_file(): + raise ProjectionError("The clean-migration website profile is not checked out") + profile = load_yaml(PROFILE_PATH) + verify_declared_pins(profile) + contract = load_projection_contract(profile) + all_records = source_closure(contract) + expectations = validate_record_contract(all_records, contract) + roundtrip_records(all_records) + + safe_reset(output) + if not PROJECTION_ATTRIBUTES.is_file(): + raise ProjectionError( + f"Projection storage policy is absent: {PROJECTION_ATTRIBUTES}" + ) + shutil.copy2(PROJECTION_ATTRIBUTES, output / ".gitattributes") + state = output / ".state" + state.mkdir() + with dump_things_service(all_records, state) as url: + exported = service_export(url, all_records, state) + (output / "records.jsonl").write_text( + "".join( + json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" + for record in exported + ), + encoding="utf-8", + ) + render_qri(url, exported, output, state, contract) + report = validate_projection(exported, output, expectations) + shutil.rmtree(state) + (output / "SHA256SUMS").write_text(projection_manifest(output), encoding="utf-8") + stack_records(exported, BUILD_ROOT / "records.jsonl") + report_path = BUILD_ROOT / "report.json" + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def compare_trees(left: Path, right: Path) -> None: + left_files = {path.relative_to(left).as_posix(): path for path in files_below(left)} + right_files = { + path.relative_to(right).as_posix(): path for path in files_below(right) + } + if left_files.keys() != right_files.keys(): + raise ProjectionError( + "Projection file sets differ: " + f"left={sorted(left_files)}, right={sorted(right_files)}" + ) + changed = [ + name + for name in left_files + if left_files[name].read_bytes() != right_files[name].read_bytes() + ] + if changed: + raise ProjectionError( + f"Projection bytes differ for: {', '.join(sorted(changed))}" + ) + + +def replace_committed(candidate: Path) -> None: + allowed = { + ".gitattributes", + "content", + "records.jsonl", + "static", + "SHA256SUMS", + } + present = {path.name for path in candidate.iterdir()} + if present != allowed: + raise ProjectionError( + f"Candidate projection paths are unexpected: {sorted(present)}" + ) + COMMITTED.mkdir(parents=True, exist_ok=True) + obsolete_records = COMMITTED / "records" + if obsolete_records.exists(): + shutil.rmtree(obsolete_records) + for name in sorted(allowed): + source = candidate / name + destination = COMMITTED / name + if destination.is_dir(): + shutil.rmtree(destination) + elif destination.exists() or destination.is_symlink(): + destination.unlink() + if source.is_dir(): + shutil.copytree(source, destination, symlinks=True) + else: + shutil.copy2(source, destination) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + render_parser = subparsers.add_parser("render") + render_parser.add_argument("--output", type=Path, default=BUILD_ROOT / "candidate") + subparsers.add_parser("update") + subparsers.add_parser("verify") + subparsers.add_parser("check-snapshot") + args = parser.parse_args() + try: + if args.command == "render": + report = render_projection(args.output) + print(json.dumps(report, sort_keys=True)) + elif args.command == "update": + candidate = BUILD_ROOT / "update" + render_projection(candidate) + replace_committed(candidate) + verify_manifest(COMMITTED) + print(f"Updated committed projection at {COMMITTED}") + elif args.command == "verify": + first = BUILD_ROOT / "verify-first" + second = BUILD_ROOT / "verify-second" + render_projection(first) + render_projection(second) + compare_trees(first, second) + compare_trees(first, COMMITTED) + verify_manifest(COMMITTED) + print("Projection rendered twice byte-identically and matches Git") + elif args.command == "check-snapshot": + verify_final_site_state(load_yaml(PROFILE_PATH)) + verify_manifest(COMMITTED) + records = [ + json.loads(line) + for line in (COMMITTED / "records.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line + ] + validate_projection(records, COMMITTED) + stack_records(records, BUILD_ROOT / "records.jsonl") + print("Committed projection digest and closure are current") + except ProjectionError as error: + print(f"clean-migration projection: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prepare_local_stack.py b/tools/prepare_local_stack.py new file mode 100755 index 0000000..0fa0f99 --- /dev/null +++ b/tools/prepare_local_stack.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Prepare the isolated local services used by the clean migration. + +The German pool cache and the CON projection are kept in separate collection +pairs. The local editor can write only to the CON incoming area. This task +only materializes runtime state under ``build/local-stack``; no records or +credentials are committed to Git. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import secrets +import shutil +import sys +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] +STACK = ROOT / "build" / "local-stack" +SNAPSHOT = STACK / "pool" / "public-thing.jsonl" +MANIFEST = STACK / "pool" / "manifest.json" +SERVICE_CONFIG = STACK / "dumpthings.yaml" +EDITOR_TOKEN = STACK / "editor-token" +SEED_TOKEN = STACK / "seed-token" +ADMIN_TOKEN = STACK / "admin-token" +POOL_UI_SOURCE = ( + ROOT / "submodules" / "pool.psychoinformatics.de-ui" / "dist" / "ui" +) +POOL_UI = STACK / "ui" +POOL_API = os.environ.get( + "UPSTREAM_POOL_API", "https://pool.psychoinformatics.de/api" +).rstrip("/") +SCHEMA = ( + ROOT + / "submodules" + / "things-schemas" + / "src" + / "demo-research-information" + / "unreleased.yaml" +) +COLLECTIONS = ( + "upstream-public", + "upstream-protected", + "con-public", + "con-protected", +) +LEGACY_COLLECTIONS = ("public", "protected") + + +def request_json(url: str, *, timeout: int = 120) -> object: + request = Request(url, headers={"Accept": "application/json"}) + for attempt in range(4): + try: + with urlopen(request, timeout=timeout) as response: + return json.load(response) + except (HTTPError, URLError, TimeoutError) as error: + if attempt == 3: + raise RuntimeError(f"Could not fetch {url}: {error}") from error + time.sleep(2**attempt) + raise AssertionError("unreachable") + + +def token_file(path: Path) -> str: + existing = path.read_text().strip() if path.exists() else "" + if existing: + path.chmod(0o600) + return existing + value = secrets.token_urlsafe(32) + path.write_text(value + "\n") + path.chmod(0o600) + return value + + +def fetch_page(page: int, size: int = 100) -> tuple[dict, int]: + """Fetch one paginated Thing page, shrinking a page that exceeds the API limit.""" + while size >= 1: + query = urlencode({"format": "json", "size": size, "page": page}) + url = f"{POOL_API}/public/records/p/Thing?{query}" + try: + result = request_json(url) + if not isinstance(result, dict) or "items" not in result: + raise RuntimeError(f"Unexpected response from {url}") + return result, size + except RuntimeError as error: + if "413" not in str(error) or size == 1: + raise + size //= 2 + raise AssertionError("unreachable") + + +def write_snapshot() -> tuple[int, dict]: + server = request_json(f"{POOL_API}/server") + size = 100 + while True: + first, size = fetch_page(1, size) + total = int(first["total"]) + pages = int(first["pages"]) + payloads = [first] + restart = False + for page in range(2, pages + 1): + payload, effective_size = fetch_page(page, size) + if effective_size != size: + size = effective_size + restart = True + break + if int(payload["total"]) != total: + raise RuntimeError( + "Upstream pool changed while its snapshot was being fetched" + ) + payloads.append(payload) + if not restart: + break + SNAPSHOT.parent.mkdir(parents=True, exist_ok=True) + temporary = SNAPSHOT.with_name(f".{SNAPSHOT.name}.tmp-{os.getpid()}") + records = 0 + seen: set[str] = set() + try: + with temporary.open("w", encoding="utf-8") as output: + for page, payload in enumerate(payloads, start=1): + for record in payload["items"]: + pid = record.get("pid") + if not isinstance(pid, str) or not pid: + raise RuntimeError( + "Upstream pool page " + f"{page} has a record without a pid" + ) + if pid in seen: + raise RuntimeError( + f"Upstream pool pagination repeated pid {pid!r}" + ) + schema_type = record.get("schema_type", "") + class_name = ( + schema_type.rsplit(":", 1)[-1] + if isinstance(schema_type, str) + else "Thing" + ) + envelope = {"class_name": class_name, "record": record} + output.write(json.dumps(envelope, sort_keys=True) + "\n") + seen.add(pid) + records += 1 + print( + f"Fetched pool page {page}/{pages} " + f"({records}/{total} records)", + flush=True, + ) + if records != total: + raise RuntimeError( + "Upstream pool snapshot is incomplete: " + f"expected {total} unique records, fetched {records}" + ) + os.replace(temporary, SNAPSHOT) + finally: + if temporary.exists(): + temporary.unlink() + return records, server if isinstance(server, dict) else {} + + +def snapshot_fingerprint(path: Path) -> tuple[int, str]: + """Validate a cached JSONL snapshot and return its count and digest.""" + digest = hashlib.sha256() + seen: set[str] = set() + with path.open("rb") as stream: + for line_number, line in enumerate(stream, start=1): + digest.update(line) + if not line.strip(): + raise RuntimeError( + f"Cached snapshot {path}:{line_number} has a blank line" + ) + try: + item = json.loads(line) + pid = item["record"]["pid"] + except (json.JSONDecodeError, KeyError, TypeError) as error: + raise RuntimeError( + f"Cached snapshot {path}:{line_number} is invalid" + ) from error + if not isinstance(pid, str) or not pid or pid in seen: + raise RuntimeError( + f"Cached snapshot {path}:{line_number} has an invalid " + f"or duplicate pid {pid!r}" + ) + seen.add(pid) + if not seen: + raise RuntimeError(f"Cached snapshot {path} has no records") + return len(seen), digest.hexdigest() + + +def yaml_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def write_service_config(editor_token: str, seed_token: str) -> None: + store = STACK / "store" + for collection in COLLECTIONS: + (store / collection / "curated").mkdir(parents=True, exist_ok=True) + (store / collection / "incoming").mkdir(parents=True, exist_ok=True) + config = f"""type: collections +version: 2 +collections: + upstream-public: + default_token: local_reader + curated: upstream-public/curated + incoming: upstream-public/incoming + schema: {yaml_quote(str(SCHEMA))} + auth_sources: + - type: config + upstream-protected: + default_token: local_reader + curated: upstream-protected/curated + incoming: upstream-protected/incoming + schema: {yaml_quote(str(SCHEMA))} + auth_sources: + - type: config + con-public: + default_token: local_reader + curated: con-public/curated + incoming: con-public/incoming + schema: {yaml_quote(str(SCHEMA))} + auth_sources: + - type: config + con-protected: + default_token: local_con_reader + curated: con-protected/curated + incoming: con-protected/incoming + schema: {yaml_quote(str(SCHEMA))} + auth_sources: + - type: config +tokens: + local_reader: + user_id: local-reader + collections: + upstream-public: + mode: READ_CURATED + upstream-protected: + mode: READ_CURATED + con-public: + mode: READ_CURATED + local_con_reader: + user_id: local-con-reader + collections: + con-protected: + mode: READ_CURATED + local_editor: + user_id: local-editor + representation: {yaml_quote(editor_token)} + collections: + con-protected: + mode: WRITE_COLLECTION + incoming_label: local-editor + local_seeder: + user_id: local-seeder + representation: {yaml_quote(seed_token)} + collections: + upstream-public: + mode: CURATOR + upstream-protected: + mode: CURATOR + con-public: + mode: CURATOR + con-protected: + mode: CURATOR +""" + SERVICE_CONFIG.write_text(config, encoding="utf-8") + SERVICE_CONFIG.chmod(0o600) + + +def reset_persisted_service_config() -> None: + """Make the service import the generated config on its next start.""" + persisted = STACK / "store" / "__dump_things__" + if persisted.exists(): + shutil.rmtree(persisted) + + +def remove_legacy_collection_stores() -> list[Path]: + """Remove only obsolete two-collection runtime stores.""" + removed: list[Path] = [] + store = STACK / "store" + for collection in LEGACY_COLLECTIONS: + path = store / collection + if path.exists(): + shutil.rmtree(path) + removed.append(path) + return removed + + +def prepare_pool_ui() -> None: + """Copy the pinned UI and specialize both service URLs for CON.""" + source_config = POOL_UI_SOURCE / "config.yaml" + if not source_config.exists(): + raise RuntimeError(f"Missing pinned pool UI configuration: {source_config}") + if POOL_UI.exists(): + shutil.rmtree(POOL_UI) + shutil.copytree(POOL_UI_SOURCE, POOL_UI) + config_path = POOL_UI / "config.yaml" + config = config_path.read_text(encoding="utf-8") + replacements = { + "http://127.0.0.1:8111/protected/": ( + "http://127.0.0.1:8111/con-protected/" + ), + "http://127.0.0.1:8111/public/": ( + "http://127.0.0.1:8111/con-protected/" + ), + } + for original, replacement in replacements.items(): + if config.count(original) != 1: + raise RuntimeError( + "Pinned pool UI service contract changed: expected one " + f"{original!r} in {source_config}" + ) + config = config.replace(original, replacement) + token_info = ( + "token_info: Please contact Michael Hanke at " + "m.hanke@fz-juelich.de for credentials." + ) + if config.count(token_info) != 1: + raise RuntimeError( + "Pinned pool UI token-information contract changed in " + f"{source_config}" + ) + config = config.replace( + token_info, + "token_info: 'Paste build/local-stack/editor-token when prompted.'", + ) + config_path.write_text(config, encoding="utf-8") + + +def main() -> int: + if not SCHEMA.exists(): + print(f"Missing local schema: {SCHEMA}", file=sys.stderr) + return 1 + STACK.mkdir(parents=True, exist_ok=True) + editor_token = token_file(EDITOR_TOKEN) + seed_token = token_file(SEED_TOKEN) + token_file(ADMIN_TOKEN) + refresh = os.environ.get("REFRESH_UPSTREAM_POOL", "") == "1" + if SNAPSHOT.exists() and MANIFEST.exists() and not refresh: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + expected_records = int(manifest["record_count"]) + records, snapshot_sha256 = snapshot_fingerprint(SNAPSHOT) + if records != expected_records: + raise RuntimeError( + "Cached upstream snapshot count does not match its manifest: " + f"expected {expected_records}, found {records}" + ) + expected_sha256 = manifest.get("snapshot_sha256") + if expected_sha256 is not None and expected_sha256 != snapshot_sha256: + raise RuntimeError( + "Cached upstream snapshot digest does not match its manifest" + ) + server = manifest.get("source_server", {}) + print( + f"Reusing {records} prepared upstream records " + "(set REFRESH_UPSTREAM_POOL=1 to refresh)" + ) + else: + records, server = write_snapshot() + verified_records, snapshot_sha256 = snapshot_fingerprint(SNAPSHOT) + if verified_records != records: + raise RuntimeError("New upstream snapshot failed its count check") + removed = remove_legacy_collection_stores() + write_service_config(editor_token, seed_token) + reset_persisted_service_config() + prepare_pool_ui() + MANIFEST.write_text( + json.dumps( + { + "source_api": POOL_API, + "source_collection": "public", + "source_class": "Thing", + "source_server": server, + "record_count": records, + "snapshot": str(SNAPSHOT.relative_to(ROOT)), + "snapshot_sha256": snapshot_sha256, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + print(f"Prepared {records} upstream records in {SNAPSHOT}") + print(f"Dump Things config: {SERVICE_CONFIG}") + print(f"Editor token: {EDITOR_TOKEN}") + print(f"Build-only seed token: {SEED_TOKEN}") + print(f"CON editor UI: {POOL_UI}") + if removed: + print( + "Removed obsolete local collection stores: " + + ", ".join(str(path) for path in removed) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/seed_local_pool.py b/tools/seed_local_pool.py new file mode 100755 index 0000000..384cbab --- /dev/null +++ b/tools/seed_local_pool.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Load isolated upstream and CON records through the local Dump Things API.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] +STACK = ROOT / "build" / "local-stack" +UPSTREAM_SNAPSHOT = STACK / "pool" / "public-thing.jsonl" +CON_RECORDS = ROOT / "build" / "con-projection" / "records.jsonl" +SEED_TOKEN = STACK / "seed-token" +SERVICE_URL = "http://127.0.0.1:8111" +CLASS_NAME = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") +UPSTREAM_COLLECTIONS = ("upstream-public", "upstream-protected") +CON_COLLECTIONS = ("con-public", "con-protected") + + +def call( + method: str, + url: str, + token: str, + body: object | None = None, +) -> tuple[int, object | None]: + headers = {"Accept": "application/json", "X-DumpThings-Token": token} + data = None + if body is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(body).encode("utf-8") + request = Request(url, headers=headers, data=data, method=method) + try: + with urlopen(request, timeout=120) as response: + raw = response.read() + return response.status, json.loads(raw) if raw else None + except HTTPError as error: + detail = error.read().decode("utf-8", errors="replace") + if error.code == 404: + return error.code, None + raise RuntimeError( + f"{method} {url} failed ({error.code}): {detail[:500]}" + ) from error + except URLError as error: + raise RuntimeError( + f"Could not reach local Dump Things service at {SERVICE_URL}: {error}" + ) from error + + +def put_record(collection: str, class_name: str, record: dict, token: str) -> str: + pid = record.get("pid") + if not isinstance(pid, str): + return "skipped" + existing_status, existing = call( + "GET", + f"{SERVICE_URL}/{collection}/curated/record?{urlencode({'pid': pid})}", + token, + ) + if existing_status == 200: + stored_record = dict(record) + stored_record.pop("schema_type", None) + if existing in (record, stored_record): + return "unchanged" + call( + "POST", + f"{SERVICE_URL}/{collection}/curated/record/{quote(class_name, safe='')}", + token, + record, + ) + return "updated" if existing is not None else "created" + + +def load_manifest(path: Path) -> list[tuple[str, dict]]: + """Load the stack JSONL envelope and reject ambiguous records.""" + records: list[tuple[str, dict]] = [] + seen: set[str] = set() + with path.open(encoding="utf-8") as stream: + for line_number, line in enumerate(stream, start=1): + if not line.strip(): + raise RuntimeError(f"{path}:{line_number}: blank JSONL line") + try: + item = json.loads(line) + except json.JSONDecodeError as error: + raise RuntimeError( + f"{path}:{line_number}: invalid JSON: {error.msg}" + ) from error + if not isinstance(item, dict): + raise RuntimeError( + f"{path}:{line_number}: expected a JSON object" + ) + class_name = item.get("class_name") + record = item.get("record") + if not isinstance(class_name, str) or not CLASS_NAME.fullmatch( + class_name + ): + raise RuntimeError( + f"{path}:{line_number}: invalid class_name {class_name!r}" + ) + if not isinstance(record, dict): + raise RuntimeError( + f"{path}:{line_number}: record must be a JSON object" + ) + pid = record.get("pid") + if not isinstance(pid, str) or not pid: + raise RuntimeError( + f"{path}:{line_number}: record must have a string pid" + ) + if pid in seen: + raise RuntimeError( + f"{path}:{line_number}: duplicate record pid {pid!r}" + ) + seen.add(pid) + records.append((class_name, record)) + if not records: + raise RuntimeError(f"{path}: manifest has no records") + return records + + +def curated_pids(collection: str, token: str) -> set[str]: + pids: set[str] = set() + page = 1 + while True: + query = urlencode({"page": page, "size": 100}) + url = f"{SERVICE_URL}/{collection}/curated/records/p/?{query}" + status, payload = call("GET", url, token) + if status != 200 or not isinstance(payload, dict): + raise RuntimeError(f"Unexpected paginated response from {url}") + items = payload.get("items") + if not isinstance(items, list): + raise RuntimeError(f"Unexpected paginated response from {url}") + for record in items: + pid = record.get("pid") if isinstance(record, dict) else None + if not isinstance(pid, str): + raise RuntimeError(f"Record without a pid in {collection}") + pids.add(pid) + pages = int(payload.get("pages", 1)) + if page >= pages: + return pids + page += 1 + + +def prune_collection( + collection: str, + expected_pids: set[str], + token: str, +) -> int: + stale = curated_pids(collection, token) - expected_pids + for pid in sorted(stale): + query = urlencode({"pid": pid}) + call( + "DELETE", + f"{SERVICE_URL}/{collection}/curated/record?{query}", + token, + ) + return len(stale) + + +def seed_manifest( + path: Path, + collections: tuple[str, str], + token: str, + label: str, +) -> dict[str, int]: + records = load_manifest(path) + counts = { + "created": 0, + "updated": 0, + "unchanged": 0, + "skipped": 0, + "deleted": 0, + } + expected_pids = {record["pid"] for _, record in records} + for collection in collections: + counts["deleted"] += prune_collection( + collection, + expected_pids, + token, + ) + targets = " and ".join(collections) + for index, (class_name, record) in enumerate(records, start=1): + for collection in collections: + result = put_record(collection, class_name, record, token) + counts[result] += 1 + if index == 1 or index % 25 == 0 or index == len(records): + print( + f"Seeded {index}/{len(records)} {label} records into {targets}", + flush=True, + ) + return counts + + +def main() -> int: + missing = [ + path + for path in (UPSTREAM_SNAPSHOT, CON_RECORDS, SEED_TOKEN) + if not path.exists() + ] + if missing: + print("Run `pixi run prepare-local-stack` first.", file=sys.stderr) + for path in missing: + print(f"Missing required local-stack input: {path}", file=sys.stderr) + return 1 + token = SEED_TOKEN.read_text(encoding="utf-8").strip() + summary = { + "upstream": seed_manifest( + UPSTREAM_SNAPSHOT, + UPSTREAM_COLLECTIONS, + token, + "upstream", + ), + "con": seed_manifest(CON_RECORDS, CON_COLLECTIONS, token, "CON"), + } + print(json.dumps(summary, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/serve_local_dumpthings.sh b/tools/serve_local_dumpthings.sh new file mode 100755 index 0000000..1f7388c --- /dev/null +++ b/tools/serve_local_dumpthings.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +stack_dir="$root_dir/build/local-stack" + +if [[ ! -f "$stack_dir/dumpthings.yaml" || ! -f "$stack_dir/admin-token" ]]; then + echo "Run 'pixi run prepare-local-stack' first." >&2 + exit 1 +fi + +export DTS_ADMIN_TOKEN="$(<"$stack_dir/admin-token")" +exec dump-things-service "$stack_dir/store" \ + --config "$stack_dir/dumpthings.yaml" \ + --host 127.0.0.1 \ + --port 8111 \ + --origins http://127.0.0.1:3000 \ + --log-level INFO diff --git a/tools/serve_local_gitannex.py b/tools/serve_local_gitannex.py new file mode 100644 index 0000000..28d720b --- /dev/null +++ b/tools/serve_local_gitannex.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Serve a small local git-annex p2p-over-HTTP repository. + +The endpoint shape is the one used by the upstream shacl-vue uploader: +``//v4/put`` for uploads and ``//key/`` for +downloads. Content is stored by git-annex itself, rather than in a separate +demo-data directory. +""" + +from __future__ import annotations + +import base64 +import os +import re +import shutil +import subprocess +import tempfile +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, unquote, urlsplit + + +ROOT = Path(__file__).resolve().parents[1] +STACK = ROOT / "build" / "local-stack" +ANNEX_REPOSITORY = STACK / "annex-repository" +ANNEX_UUID = os.environ.get( + "LOCAL_ANNEX_UUID", "00000000-0000-0000-0000-000000000001" +) +HOST = os.environ.get("LOCAL_ANNEX_HOST", "127.0.0.1") +PORT = int(os.environ.get("LOCAL_ANNEX_PORT", "8122")) +EDITOR_TOKEN_PATH = STACK / "editor-token" +KEY_RE = re.compile(r"^SHA256E-s[0-9]+--[0-9a-f]+(?:\.[A-Za-z0-9._-]+)?$") + + +def run_annex(*args: str) -> str: + result = subprocess.run( + ["git", "-C", str(ANNEX_REPOSITORY), "annex", *args], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def ensure_repository() -> None: + ANNEX_REPOSITORY.mkdir(parents=True, exist_ok=True) + if not (ANNEX_REPOSITORY / ".git" / "annex").exists(): + subprocess.run(["git", "-C", str(ANNEX_REPOSITORY), "init", "-q"], check=True) + run_annex("init", "local deployment") + + +def content_location(key: str) -> Path | None: + try: + location = run_annex("contentlocation", key) + except subprocess.CalledProcessError: + # git-annex returns non-zero when the key is not known locally. + return None + if not location: + return None + path = ANNEX_REPOSITORY / location + return path if path.exists() else None + + +class Handler(BaseHTTPRequestHandler): + server_version = "OrinocoLocalGitAnnex/1.0" + + def _headers(self, status: HTTPStatus, length: int = 0, content_type: str = "text/plain") -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(length)) + self.send_header("Access-Control-Allow-Origin", "http://127.0.0.1:3000") + self.send_header("Access-Control-Allow-Headers", "Authorization, Content-Type, X-git-annex-data-length") + self.send_header("Access-Control-Allow-Methods", "GET, HEAD, OPTIONS, POST") + self.end_headers() + + def _write(self, status: HTTPStatus, body: bytes = b"") -> None: + self._headers(status, len(body)) + if self.command != "HEAD": + self.wfile.write(body) + + def _authorized(self) -> bool: + expected = EDITOR_TOKEN_PATH.read_text(encoding="utf-8").strip() + value = self.headers.get("Authorization", "") + if not value.startswith("Basic "): + return False + try: + decoded = base64.b64decode(value[6:]).decode("utf-8") + except (ValueError, UnicodeDecodeError): + return False + supplied, _, _ = decoded.partition(":") + return supplied == expected + + def _key_from_path(self) -> str | None: + prefix = f"/git-annex/{ANNEX_UUID}/key/" + path = unquote(urlsplit(self.path).path) + if not path.startswith(prefix): + return None + key = path[len(prefix) :] + return key if KEY_RE.fullmatch(key) else None + + def do_OPTIONS(self) -> None: # noqa: N802 + self._write(HTTPStatus.NO_CONTENT) + + def do_HEAD(self) -> None: # noqa: N802 + self._serve_key(head_only=True) + + def do_GET(self) -> None: # noqa: N802 + self._serve_key(head_only=False) + + def _serve_key(self, *, head_only: bool) -> None: + key = self._key_from_path() + if key is None: + self._write(HTTPStatus.NOT_FOUND, b"Unknown git-annex key\n") + return + if not self._authorized(): + self._write(HTTPStatus.UNAUTHORIZED, b"A local editor token is required\n") + return + path = content_location(key) + if path is None: + self._write(HTTPStatus.NOT_FOUND, b"Key is not present\n") + return + data_size = path.stat().st_size + self._headers(HTTPStatus.OK, data_size, "application/octet-stream") + if not head_only: + with path.open("rb") as source: + shutil.copyfileobj(source, self.wfile) + + def do_POST(self) -> None: # noqa: N802 + prefix = f"/git-annex/{ANNEX_UUID}/v4/put" + if urlsplit(self.path).path != prefix: + self._write(HTTPStatus.NOT_FOUND, b"Unknown git-annex endpoint\n") + return + if not self._authorized(): + self._write(HTTPStatus.UNAUTHORIZED, b"A local editor token is required\n") + return + key_values = parse_qs(urlsplit(self.path).query).get("key", []) + key = key_values[0] if key_values else "" + if not KEY_RE.fullmatch(key): + self._write(HTTPStatus.BAD_REQUEST, b"A SHA256E git-annex key is required\n") + return + length = int(self.headers.get("Content-Length", "0")) + if length <= 0: + self._write(HTTPStatus.BAD_REQUEST, b"Content-Length is required\n") + return + existing = content_location(key) + if existing is not None and existing.stat().st_size == length: + self._write(HTTPStatus.OK, b"already present\n") + return + with tempfile.NamedTemporaryFile( + dir=ANNEX_REPOSITORY, prefix="upload-", suffix=Path(key).suffix, delete=False + ) as upload: + temporary_path = Path(upload.name) + remaining = length + while remaining: + chunk = self.rfile.read(min(1024 * 1024, remaining)) + if not chunk: + break + upload.write(chunk) + remaining -= len(chunk) + if remaining: + temporary_path.unlink(missing_ok=True) + self._write(HTTPStatus.BAD_REQUEST, b"Upload ended before Content-Length\n") + return + named_path = ANNEX_REPOSITORY / ("upload-" + Path(key).name) + temporary_path.replace(named_path) + try: + run_annex("add", "--backend=SHA256E", "--force", str(named_path.relative_to(ANNEX_REPOSITORY))) + actual_key = run_annex("lookupkey", str(named_path.relative_to(ANNEX_REPOSITORY))) + if actual_key != key: + self._write(HTTPStatus.BAD_REQUEST, b"Content does not match the requested key\n") + return + finally: + named_path.unlink(missing_ok=True) + self._write(HTTPStatus.OK, b"stored\n") + + def log_message(self, format: str, *args: object) -> None: + print(f"local-git-annex: {format % args}", flush=True) + + +def main() -> int: + ensure_repository() + if not EDITOR_TOKEN_PATH.exists(): + raise SystemExit("Run `pixi run prepare-local-stack` first.") + server = ThreadingHTTPServer((HOST, PORT), Handler) + print(f"Local git-annex p2p service listening at http://{HOST}:{PORT}/git-annex", flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + return 0 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/serve_local_stack.sh b/tools/serve_local_stack.sh new file mode 100755 index 0000000..2fd142a --- /dev/null +++ b/tools/serve_local_stack.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +stack_dir="$root_dir/build/local-stack" +log_dir="$stack_dir/logs" +mkdir -p "$log_dir" + +pids=() +names=() + +cleanup() { + trap - EXIT INT TERM + for pid in "${pids[@]}"; do + kill "$pid" 2>/dev/null || true + done + for pid in "${pids[@]}"; do + wait "$pid" 2>/dev/null || true + done +} +trap cleanup EXIT +trap 'exit 130' INT TERM + +for required_dir in "$stack_dir/ui" "$root_dir/build/con-site"; do + if [[ ! -d "$required_dir" ]]; then + echo "Missing local-stack input directory: $required_dir" >&2 + exit 1 + fi +done + +start_background() { + local name="$1" + shift + echo "Starting $name (log: $log_dir/$name.log)" + "$@" >"$log_dir/$name.log" 2>&1 & + pids+=("$!") + names+=("$name") +} + +wait_for_url() { + local name="$1" + local url="$2" + local attempts=0 + while (( attempts < 120 )); do + if python3 - "$url" <<'PY' +import sys +from urllib.request import urlopen + +try: + with urlopen(sys.argv[1], timeout=2): + pass +except Exception: + raise SystemExit(1) +PY + then + echo "$name is ready: $url" + return 0 + fi + ((attempts += 1)) + sleep 1 + done + echo "$name did not become ready: $url" >&2 + return 1 +} + +start_background dump-things "$root_dir/tools/serve_local_dumpthings.sh" +wait_for_url "Dump Things" "http://127.0.0.1:8111/server" + +python3 "$root_dir/tools/seed_local_pool.py" + +start_background git-annex python3 "$root_dir/tools/serve_local_gitannex.py" +start_background shacl-vue python3 -m http.server 3000 \ + --directory "$stack_dir/ui" +wait_for_url "SHACL Vue" "http://127.0.0.1:3000/config.yaml" +python3 "$root_dir/tools/check_local_stack.py" + +start_background con-site python3 -m http.server 8767 \ + --directory "$root_dir/build/con-site" +wait_for_url "CON site" "http://127.0.0.1:8767/" +echo "Local deployment is ready at http://127.0.0.1:8767/" +echo "Press Ctrl-C to stop all local services." + +while :; do + for index in "${!pids[@]}"; do + if ! kill -0 "${pids[$index]}" 2>/dev/null; then + echo "${names[$index]} exited unexpectedly; see $log_dir/${names[$index]}.log" >&2 + exit 1 + fi + done + sleep 1 +done