From 9c7c5228a11ca61ec2c4115c45f0e8f3966613a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martynas=20Jusevi=C4=8Dius?= Date: Fri, 16 Oct 2026 10:23:26 +0200 Subject: [PATCH 1/2] Each app has a sync.sh beside its install.sh that can run on every commit: it pushes the tree when a pushed file changed, re-imports the ontology only when the model changed, re-runs only the imports whose files changed and waits for them to finish, and deletes the documents whose sources are gone. A master push that passes check-docs dispatches its commit range to linkeddatahub.com, which holds the credentials and runs the syncs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check-docs.yml | 20 +++++ demo/copenhagen/sync.sh | 28 +++++++ demo/northwind-traders/sync.sh | 28 +++++++ demo/unesco-thesaurus/sync.sh | 29 ++++++++ docs/sync.sh | 17 +++++ lib/ldh-app.sh | 123 ++++++++++++++++++++++++++++++- packages/sync.sh | 28 +++++++ 7 files changed, 271 insertions(+), 2 deletions(-) create mode 100755 demo/copenhagen/sync.sh create mode 100755 demo/northwind-traders/sync.sh create mode 100755 demo/unesco-thesaurus/sync.sh create mode 100755 docs/sync.sh create mode 100755 packages/sync.sh diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index 70c0eaf3..1967280e 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -36,3 +36,23 @@ jobs: - name: Check links run: ./check-links.sh + + # A master push that passes the checks goes live: linkeddatahub.com's sync-dataspaces workflow + # holds the deployment's credentials and maps the changed paths onto its dataspaces + dispatch: + name: Sync linkeddatahub.com + needs: check-docs + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + + steps: + - name: Dispatch the commit range + env: + GH_TOKEN: ${{ secrets.SITE_DISPATCH_TOKEN }} # fine-grained, contents: write on AtomGraph/linkeddatahub.com only + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + run: | + gh api repos/AtomGraph/linkeddatahub.com/dispatches \ + -f event_type=apps-pushed \ + -f "client_payload[before]=$BEFORE" \ + -f "client_payload[after]=$AFTER" diff --git a/demo/copenhagen/sync.sh b/demo/copenhagen/sync.sh new file mode 100755 index 00000000..96689778 --- /dev/null +++ b/demo/copenhagen/sync.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports +# the namespace ontology if the model changed, pushes the document tree if a pushed document or file +# changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or +# data changed, waiting for them to finish. Safe to run on every commit; install.sh does the one-off +# steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +if ldh_app_changed 'admin/model/*'; then + ldh_app_step "Importing namespace ontology" + ldh_app_import_ns "$app_dir/admin/model" +fi + +ldh_app_step "Pushing changed documents and files" +ldh_app_push_changed "$app_dir" + +ldh_app_step "Deleting removed documents" +ldh_app_delete_removed "$app_dir" + +ldh_app_step "Importing changed CSV data" +ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" +ldh_app_await_imports diff --git a/demo/northwind-traders/sync.sh b/demo/northwind-traders/sync.sh new file mode 100755 index 00000000..96689778 --- /dev/null +++ b/demo/northwind-traders/sync.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports +# the namespace ontology if the model changed, pushes the document tree if a pushed document or file +# changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or +# data changed, waiting for them to finish. Safe to run on every commit; install.sh does the one-off +# steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +if ldh_app_changed 'admin/model/*'; then + ldh_app_step "Importing namespace ontology" + ldh_app_import_ns "$app_dir/admin/model" +fi + +ldh_app_step "Pushing changed documents and files" +ldh_app_push_changed "$app_dir" + +ldh_app_step "Deleting removed documents" +ldh_app_delete_removed "$app_dir" + +ldh_app_step "Importing changed CSV data" +ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" +ldh_app_await_imports diff --git a/demo/unesco-thesaurus/sync.sh b/demo/unesco-thesaurus/sync.sh new file mode 100755 index 00000000..03116a25 --- /dev/null +++ b/demo/unesco-thesaurus/sync.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: pushes the +# document tree if a pushed document or file changed, deletes the documents whose sources are gone, +# and re-imports the UNESCO Thesaurus if its data or query changed, waiting for it to finish. Safe to +# run on every commit; install.sh does the one-off steps, the package import among them. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_step "Pushing changed documents and files" +ldh_app_push_changed "$app_dir" + +ldh_app_step "Deleting removed documents" +ldh_app_delete_removed "$app_dir" + +if ldh_app_changed 'concept-schemes/unesco-thesaurus/*'; then + ldh_app_step "Importing SKOS vocabulary" + ldh_app_track_import "${LDH_BASE}concept-schemes/unesco-thesaurus/" ldh import rdf \ + --title "Unesco Thesaurus SKOS" \ + --query-file "$app_dir/concept-schemes/unesco-thesaurus/skos-import.rq" \ + --rdf-file "$app_dir/concept-schemes/unesco-thesaurus/unesco-thesaurus.ttl" \ + --content-type "text/turtle" \ + "${LDH_BASE}concept-schemes/unesco-thesaurus/" < /dev/null + ldh_app_await_imports +fi diff --git a/docs/sync.sh b/docs/sync.sh new file mode 100755 index 00000000..95a953cc --- /dev/null +++ b/docs/sync.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Brings the documentation on a LinkedDataHub instance up to date with this folder after a commit: +# pushes the document tree if a pushed document or file changed and deletes the documents whose +# sources are gone. Safe to run on every commit; install.sh does the one-off steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_step "Pushing changed documents and media" +ldh_app_push_changed "$app_dir" + +ldh_app_step "Deleting removed documents" +ldh_app_delete_removed "$app_dir" diff --git a/lib/ldh-app.sh b/lib/ldh-app.sh index 48819be8..35655288 100755 --- a/lib/ldh-app.sh +++ b/lib/ldh-app.sh @@ -60,17 +60,136 @@ ldh_app_import_ns() { # Runs one ldh import csv per row of the manifest $2 — query_filename,csv_filename,target,title, paths # relative to $1: it adds the CONSTRUCT query, uploads the CSV into the target document and creates # the import that maps one into the other. Every run creates a new import; the documents the imports -# write are the same, so the data converges. +# write are the same, so the data converges. A sync runs only the rows whose query or CSV changed. ldh_app_import_csv() { local app_dir="$1" manifest="$2" local query_filename csv_filename target_path title while IFS=, read -r query_filename csv_filename target_path title; do [ -n "$query_filename" ] || continue + ldh_app_changed "$query_filename" "$csv_filename" || continue printf "Importing %s into %s\n" "$csv_filename" "${LDH_BASE}${target_path}" - ldh import csv \ + ldh_app_track_import "${LDH_BASE}${target_path}" ldh import csv \ --title "$title" \ --query-file "$app_dir/$query_filename" \ --csv-file "$app_dir/$csv_filename" \ "${LDH_BASE}${target_path}" < /dev/null done < <(tail -n +2 "$manifest") } + +# --- sync ------------------------------------------------------------------------------------------- +# +# A sync.sh redoes only what the changed files require, so it can run on every commit; install.sh +# stays the one-off. Two variables describe the commit, both optional: +# +# LDH_CHANGED newline-separated paths, relative to the app folder, added or modified by it. +# Unset means everything changed: a manual `make sync` is a full sync. +# LDH_PREVIOUS_TREE the app folder as it was before the commit. When set, documents its files +# mapped to and the current files no longer do are deleted. + +# True when any of the patterns (globs relative to the app folder, * crossing slashes) matches a +# changed path, and always when LDH_CHANGED is unset. +ldh_app_changed() { + [ -n "${LDH_CHANGED+set}" ] || return 0 + local path pattern + while IFS= read -r path; do + [ -n "$path" ] || continue + for pattern in "$@"; do + # shellcheck disable=SC2053 # the pattern is a glob on purpose + [[ "$path" == $pattern ]] && return 0 + done + done <<< "$LDH_CHANGED" + return 1 +} + +# The plan `ldh push` makes for folder $1 onto $LDH_BASE, one "METHOD URI <- relative path" line per +# step. A dry run reads the files and sends nothing. +ldh_app_push_plan() { + ldh push --dry-run --dir "$1" "$LDH_BASE" 2>&1 >/dev/null | grep -E '^(PUT|POST) ' || true +} + +# Pushes app folder $1 when a changed file is part of its push plan: a document it PUTs or a file it +# uploads. Files the push ignores (.ldhignore) never trigger it. +ldh_app_push_changed() { + local app_dir="$1" method uri arrow relative + if [ -z "${LDH_CHANGED+set}" ]; then + ldh push --dir "$app_dir" "$LDH_BASE" + return + fi + while read -r method uri arrow relative; do + if ldh_app_changed "$relative"; then + ldh push --dir "$app_dir" "$LDH_BASE" < /dev/null + return + fi + done < <(ldh_app_push_plan "$app_dir") + echo "No pushed document or file changed" +} + +# Deletes the documents app folder $1 no longer maps to: those in the push plan of LDH_PREVIOUS_TREE +# that are not in the current one, children before their parents. Uploads are kept, since other +# documents may still link them, and so are documents an import wrote, which no plan contains. +ldh_app_delete_removed() { + local app_dir="$1" uri + [ -n "${LDH_PREVIOUS_TREE:-}" ] || return 0 + [ -d "$LDH_PREVIOUS_TREE" ] || return 0 + while IFS= read -r uri; do + [ -n "$uri" ] || continue + printf "Deleting %s\n" "$uri" + ldh delete "$uri" < /dev/null + done < <(comm -23 \ + <(ldh_app_push_plan "$LDH_PREVIOUS_TREE" | awk '$1 == "PUT" { print $2 }' | sort -u) \ + <(ldh_app_push_plan "$app_dir" | awk '$1 == "PUT" { print $2 }' | sort -u) \ + | sort -r) +} + +# The ldh:CSVImport and ldh:RDFImport resources in document $1. +ldh_app_import_uris() { + ldh get --accept application/n-triples "$1" | + grep -E ' ' | + awk '{ print $1 }' | tr -d '<>' | sort -u || true +} + +# Runs the import command "${@:2}" into document $1 and records the import it creates in +# LDH_APP_IMPORTS, for ldh_app_await_imports. +LDH_APP_IMPORTS=() +ldh_app_track_import() { + local doc="$1" before uri + shift + before=$(ldh_app_import_uris "$doc") + "$@" + while IFS= read -r uri; do + [ -n "$uri" ] && LDH_APP_IMPORTS+=("$uri") + done < <(comm -13 <(echo "$before") <(ldh_app_import_uris "$doc")) +} + +# Waits until every import in LDH_APP_IMPORTS carries prov:endedAtTime, which the platform adds to the +# import's own document when it finishes. Fails when one of them reports an error (a prov:Entity with +# a dct:description) or when LDH_IMPORT_TIMEOUT seconds (default 1200) pass first. +ldh_app_await_imports() { + [ "${#LDH_APP_IMPORTS[@]}" -gt 0 ] || return 0 + local deadline=$(( $(date +%s) + ${LDH_IMPORT_TIMEOUT:-1200} )) + local uri doc triples errors failed=0 + for uri in "${LDH_APP_IMPORTS[@]}"; do + doc="${uri%%#*}" + printf "Waiting for %s\n" "$uri" + while :; do + triples=$(ldh get --accept application/n-triples "$doc") + grep -qF "<$uri> " <<< "$triples" && break + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Timed out waiting for $uri" >&2 + return 1 + fi + sleep 10 + done + # the error entities point at the import with prov:wasGeneratedBy + errors=$(grep -F " <$uri> " <<< "$triples" | awk '{ print $1 }' | + while IFS= read -r entity; do + grep -F "$entity " <<< "$triples" || true + done) + if [ -n "$errors" ]; then + echo "Import failed: $uri" >&2 + echo "$errors" >&2 + failed=1 + fi + done + return "$failed" +} diff --git a/packages/sync.sh b/packages/sync.sh new file mode 100755 index 00000000..e19aea58 --- /dev/null +++ b/packages/sync.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Brings the package registry on a LinkedDataHub instance up to date with this folder after a commit: +# pushes the package descriptors and files if one changed, deletes the documents whose sources are +# gone and re-declares the stylesheet of every package whose stylesheet changed. Dataspaces that +# already import a package keep the stylesheet they copied until the package is added again. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_step "Pushing changed package descriptors and files" +ldh_app_push_changed "$app_dir" + +ldh_app_step "Deleting removed documents" +ldh_app_delete_removed "$app_dir" + +ldh_app_step "Declaring changed package stylesheets" +(cd "$app_dir" && find . -name '*.xsl' -not -path './.*' | sed 's|^\./||' | sort) | while read -r stylesheet; do + ldh_app_changed "$stylesheet" || continue + package_doc="${LDH_BASE}$(dirname "$stylesheet")/" + upload="${LDH_BASE}uploads/$(shasum -a 1 "$app_dir/$stylesheet" | cut -d' ' -f1)" + printf "Declaring %s as the stylesheet of %s#this\n" "$upload" "$package_doc" + echo "INSERT { <${package_doc}#this> <${upload}> } WHERE { }" | + ldh patch "$package_doc" +done From d36673679d4637e1cd75f3b29030d3cc2f74c77b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martynas=20Jusevi=C4=8Dius?= Date: Sat, 17 Oct 2026 16:07:14 +0200 Subject: [PATCH 2/2] sync.sh stays with the apps and the trigger moves to linkeddatahub.com The check-docs dispatch job goes: it named one deployment and needed that deployment's secret, and a dispatched commit range was lost when the sync it triggered failed. linkeddatahub.com now polls for new commits against its own last-synced marker. Imports in a sync pass --wait, which develop's CLI calls already carry, so ldh_app_track_import and ldh_app_await_imports go. ldh_app_sync covers an app made of an ontology, a document tree and an imports.csv; copenhagen, northwind-traders and docs are that call, and unesco-thesaurus and packages add their own step after it. packages re-declares every stylesheet whenever the push ran, since the push PUTs every descriptor and drops the ac:stylesheet install.sh added, not only when a .xsl changed. `make sync` runs a full sync with the same prompts as `make install`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check-docs.yml | 20 ------ CLAUDE.md | 6 +- demo/copenhagen/sync.sh | 17 +---- demo/northwind-traders/sync.sh | 17 +---- demo/unesco-thesaurus/sync.sh | 14 ++--- docs/sync.sh | 6 +- lib/install.mk | 12 ++-- lib/ldh-app.sh | 104 +++++++++++-------------------- packages/sync.sh | 15 +++-- 9 files changed, 66 insertions(+), 145 deletions(-) diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index 1967280e..70c0eaf3 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -36,23 +36,3 @@ jobs: - name: Check links run: ./check-links.sh - - # A master push that passes the checks goes live: linkeddatahub.com's sync-dataspaces workflow - # holds the deployment's credentials and maps the changed paths onto its dataspaces - dispatch: - name: Sync linkeddatahub.com - needs: check-docs - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest - - steps: - - name: Dispatch the commit range - env: - GH_TOKEN: ${{ secrets.SITE_DISPATCH_TOKEN }} # fine-grained, contents: write on AtomGraph/linkeddatahub.com only - BEFORE: ${{ github.event.before }} - AFTER: ${{ github.sha }} - run: | - gh api repos/AtomGraph/linkeddatahub.com/dispatches \ - -f event_type=apps-pushed \ - -f "client_payload[before]=$BEFORE" \ - -f "client_payload[after]=$AFTER" diff --git a/CLAUDE.md b/CLAUDE.md index 03567b69..d8f95f71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ LinkedDataHub-Apps holds dataspaces built on LinkedDataHub. Every dataspace is a - `demo/copenhagen/` — Copenhagen open data on a map, imported from CSV - `demo/unesco-thesaurus/` — the UNESCO Thesaurus as a SKOS editor; its model and stylesheet come from the taxonomy editor package - `packages/` — the package registry published at https://packages.linkeddatahub.com/; one package so far, `editor/taxonomy` -- `lib/` — the shared installer: `install.mk` (the interactive `make install`) and `ldh-app.sh` (functions every `install.sh` sources) +- `lib/` — the shared installer: `install.mk` (the interactive `make install` and `make sync`) and `ldh-app.sh` (functions every `install.sh` and `sync.sh` sources) - `screencast/` — the Playwright rig that produces the docs' screenshots and clips, and the Northwind screencast tracks - `check-links.sh`, `check-xmlliterals.sh` — repository-wide validators, run by CI @@ -43,6 +43,10 @@ Each `install.sh` sources `lib/ldh-app.sh` and is the app's own sequence of step **Re-runs converge but are not clean.** `ldh push` PUTs and the ontology is reset before re-import, so documents and the model end up the same. `make-public`, `create authorization` and every import are POSTs, so each run adds another authorization and another import record. Nothing probes for an earlier install. +### Syncing an app + +`sync.sh`, beside each `install.sh`, brings an installed app up to date and is safe to run on every commit. It skips the one-off steps (authorizations, `packages add`), re-imports the namespace ontology only when `admin/model/` changed, pushes only when a file in the push plan changed (`ldh push --dry-run`, so `.ldhignore` holds), re-runs only the import rows whose query or CSV changed, each with `--wait`, and deletes the documents whose sources are gone. Two optional variables describe the commit: `LDH_CHANGED`, the added or modified paths relative to the app folder, and `LDH_PREVIOUS_TREE`, the app folder as it was before. With neither, `make sync` is a full sync. Mapping commit ranges onto apps is the deployment's job; linkeddatahub.com does it in its own repository. + ### App layout The file tree is the URL tree. `categories.ttl` installs to `{base}categories/`, the sibling folder `categories/` holds that document's uploads (and, by convention, its `.csv` and mapping `.rq`), and `root.ttl` is the base document itself. `.ldhignore` lists what is neither a document nor an upload — `admin/`, `Makefile`, `*.sh`, `*.csv`, repository screenshots — because the CLI skips nothing but hidden entries. diff --git a/demo/copenhagen/sync.sh b/demo/copenhagen/sync.sh index 96689778..5cc398f3 100755 --- a/demo/copenhagen/sync.sh +++ b/demo/copenhagen/sync.sh @@ -2,7 +2,7 @@ # Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports # the namespace ontology if the model changed, pushes the document tree if a pushed document or file # changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or -# data changed, waiting for them to finish. Safe to run on every commit; install.sh does the one-off +# data changed, waiting for each to end. Safe to run on every commit; install.sh does the one-off # steps. # # Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). @@ -12,17 +12,4 @@ app_dir="$(cd "$(dirname "$0")" && pwd)" . "$app_dir/../../lib/ldh-app.sh" ldh_app_require_env -if ldh_app_changed 'admin/model/*'; then - ldh_app_step "Importing namespace ontology" - ldh_app_import_ns "$app_dir/admin/model" -fi - -ldh_app_step "Pushing changed documents and files" -ldh_app_push_changed "$app_dir" - -ldh_app_step "Deleting removed documents" -ldh_app_delete_removed "$app_dir" - -ldh_app_step "Importing changed CSV data" -ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" -ldh_app_await_imports +ldh_app_sync "$app_dir" diff --git a/demo/northwind-traders/sync.sh b/demo/northwind-traders/sync.sh index 96689778..5cc398f3 100755 --- a/demo/northwind-traders/sync.sh +++ b/demo/northwind-traders/sync.sh @@ -2,7 +2,7 @@ # Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports # the namespace ontology if the model changed, pushes the document tree if a pushed document or file # changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or -# data changed, waiting for them to finish. Safe to run on every commit; install.sh does the one-off +# data changed, waiting for each to end. Safe to run on every commit; install.sh does the one-off # steps. # # Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). @@ -12,17 +12,4 @@ app_dir="$(cd "$(dirname "$0")" && pwd)" . "$app_dir/../../lib/ldh-app.sh" ldh_app_require_env -if ldh_app_changed 'admin/model/*'; then - ldh_app_step "Importing namespace ontology" - ldh_app_import_ns "$app_dir/admin/model" -fi - -ldh_app_step "Pushing changed documents and files" -ldh_app_push_changed "$app_dir" - -ldh_app_step "Deleting removed documents" -ldh_app_delete_removed "$app_dir" - -ldh_app_step "Importing changed CSV data" -ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" -ldh_app_await_imports +ldh_app_sync "$app_dir" diff --git a/demo/unesco-thesaurus/sync.sh b/demo/unesco-thesaurus/sync.sh index 03116a25..2cb0790d 100755 --- a/demo/unesco-thesaurus/sync.sh +++ b/demo/unesco-thesaurus/sync.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Brings the app on a LinkedDataHub instance up to date with this folder after a commit: pushes the # document tree if a pushed document or file changed, deletes the documents whose sources are gone, -# and re-imports the UNESCO Thesaurus if its data or query changed, waiting for it to finish. Safe to -# run on every commit; install.sh does the one-off steps, the package import among them. +# and re-imports the UNESCO Thesaurus if its data or query changed, waiting for it to end. Safe to run +# on every commit; install.sh does the one-off steps, the package import among them. # # Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). set -euo pipefail @@ -11,19 +11,15 @@ app_dir="$(cd "$(dirname "$0")" && pwd)" . "$app_dir/../../lib/ldh-app.sh" ldh_app_require_env -ldh_app_step "Pushing changed documents and files" -ldh_app_push_changed "$app_dir" - -ldh_app_step "Deleting removed documents" -ldh_app_delete_removed "$app_dir" +ldh_app_sync "$app_dir" if ldh_app_changed 'concept-schemes/unesco-thesaurus/*'; then ldh_app_step "Importing SKOS vocabulary" - ldh_app_track_import "${LDH_BASE}concept-schemes/unesco-thesaurus/" ldh import rdf \ + ldh import rdf \ --title "Unesco Thesaurus SKOS" \ --query-file "$app_dir/concept-schemes/unesco-thesaurus/skos-import.rq" \ --rdf-file "$app_dir/concept-schemes/unesco-thesaurus/unesco-thesaurus.ttl" \ --content-type "text/turtle" \ + --wait \ "${LDH_BASE}concept-schemes/unesco-thesaurus/" < /dev/null - ldh_app_await_imports fi diff --git a/docs/sync.sh b/docs/sync.sh index 95a953cc..19f486fd 100755 --- a/docs/sync.sh +++ b/docs/sync.sh @@ -10,8 +10,4 @@ app_dir="$(cd "$(dirname "$0")" && pwd)" . "$app_dir/../lib/ldh-app.sh" ldh_app_require_env -ldh_app_step "Pushing changed documents and media" -ldh_app_push_changed "$app_dir" - -ldh_app_step "Deleting removed documents" -ldh_app_delete_removed "$app_dir" +ldh_app_sync "$app_dir" diff --git a/lib/install.mk b/lib/install.mk index 363ce0da..bf77455a 100644 --- a/lib/install.mk +++ b/lib/install.mk @@ -1,14 +1,14 @@ # Shared by every app Makefile. `make install` prompts for the dataspace and the owner's certificate, # exports them as the LDH_* variables the ldh CLI reads, and runs the app's install.sh in that -# environment. No value travels on a command line, so the password needs no escaping and is not -# visible in the process list. +# environment; `make sync` does the same for sync.sh, a full sync. No value travels on a command line, +# so the password needs no escaping and is not visible in the process list. # # An app Makefile sets the defaults it wants to offer, then includes this file: # # CERT_PATH_DEFAULT = ../../../LinkedDataHub/ssl/owner/keystore.p12 # include ../../lib/install.mk # -# To install unattended, export the variables and run ./install.sh yourself. +# To install or sync unattended, export the variables and run ./install.sh or ./sync.sh yourself. SHELL := /bin/bash @@ -16,9 +16,9 @@ BASE_URL_DEFAULT ?= https://localhost:4443/ CERT_PATH_DEFAULT ?= ../../LinkedDataHub/ssl/owner/keystore.p12 PROXY_URL_DEFAULT ?= https://localhost:5443/ -.PHONY: install +.PHONY: install sync -install: +install sync: @read -p "Enter Base URL [$(BASE_URL_DEFAULT)]: " BASE_URL; \ read -p "Enter Certificate Path [$(CERT_PATH_DEFAULT)]: " CERT_PATH; \ read -s -p "Enter Certificate Password (required): " PASSWORD; \ @@ -32,4 +32,4 @@ install: export LDH_CERT_FILE="$${CERT_PATH:-$(CERT_PATH_DEFAULT)}"; \ export LDH_CERT_PASSWORD="$${PASSWORD}"; \ if [ -n "$${PROXY_URL}" ]; then export LDH_PROXY="$${PROXY_URL}"; fi; \ - ./install.sh + ./$@.sh diff --git a/lib/ldh-app.sh b/lib/ldh-app.sh index 7deacaf2..7bc0d134 100755 --- a/lib/ldh-app.sh +++ b/lib/ldh-app.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Shared by every install.sh in this repository. Sourced, not run. +# Shared by every install.sh and sync.sh in this repository. Sourced, not run. # # The ldh CLI reads its connection from the environment — LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD # and LDH_PROXY — so the installers put no credential on any command line: nothing to escape, nothing @@ -46,7 +46,7 @@ ldh_app_import_csv() { [ -n "$query_filename" ] || continue ldh_app_changed "$query_filename" "$csv_filename" || continue printf "Importing %s into %s\n" "$csv_filename" "${LDH_BASE}${target_path}" - ldh_app_track_import "${LDH_BASE}${target_path}" ldh import csv \ + ldh import csv \ --title "$title" \ --query-file "$app_dir/$query_filename" \ --csv-file "$app_dir/$csv_filename" \ @@ -58,10 +58,10 @@ ldh_app_import_csv() { # --- sync ------------------------------------------------------------------------------------------- # # A sync.sh redoes only what the changed files require, so it can run on every commit; install.sh -# stays the one-off. Two variables describe the commit, both optional: +# stays the one-off (authorizations, packages). Two variables describe the commit, both optional: # # LDH_CHANGED newline-separated paths, relative to the app folder, added or modified by it. -# Unset means everything changed: a manual `make sync` is a full sync. +# Unset means everything changed: `make sync` with neither set is a full sync. # LDH_PREVIOUS_TREE the app folder as it was before the commit. When set, documents its files # mapped to and the current files no longer do are deleted. @@ -87,20 +87,23 @@ ldh_app_push_plan() { } # Pushes app folder $1 when a changed file is part of its push plan: a document it PUTs or a file it -# uploads. Files the push ignores (.ldhignore) never trigger it. +# uploads. Files the push ignores (.ldhignore) never trigger it. The push PUTs every document, not only +# the changed ones, so LDH_APP_PUSHED is set to 1 for whatever a sync must add back afterwards. +LDH_APP_PUSHED=0 ldh_app_push_changed() { local app_dir="$1" method uri arrow relative - if [ -z "${LDH_CHANGED+set}" ]; then - ldh push --dir "$app_dir" "$LDH_BASE" - return - fi - while read -r method uri arrow relative; do - if ldh_app_changed "$relative"; then - ldh push --dir "$app_dir" "$LDH_BASE" < /dev/null - return + if [ -n "${LDH_CHANGED+set}" ]; then + while read -r method uri arrow relative; do + ldh_app_changed "$relative" && break + relative="" + done < <(ldh_app_push_plan "$app_dir") + if [ -z "$relative" ]; then + echo "No pushed document or file changed" + return 0 fi - done < <(ldh_app_push_plan "$app_dir") - echo "No pushed document or file changed" + fi + ldh push --dir "$app_dir" "$LDH_BASE" < /dev/null + LDH_APP_PUSHED=1 } # Deletes the documents app folder $1 no longer maps to: those in the push plan of LDH_PREVIOUS_TREE @@ -108,8 +111,7 @@ ldh_app_push_changed() { # documents may still link them, and so are documents an import wrote, which no plan contains. ldh_app_delete_removed() { local app_dir="$1" uri - [ -n "${LDH_PREVIOUS_TREE:-}" ] || return 0 - [ -d "$LDH_PREVIOUS_TREE" ] || return 0 + [ -n "${LDH_PREVIOUS_TREE:-}" ] && [ -d "$LDH_PREVIOUS_TREE" ] || return 0 while IFS= read -r uri; do [ -n "$uri" ] || continue printf "Deleting %s\n" "$uri" @@ -120,55 +122,25 @@ ldh_app_delete_removed() { | sort -r) } -# The ldh:CSVImport and ldh:RDFImport resources in document $1. -ldh_app_import_uris() { - ldh get --accept application/n-triples "$1" | - grep -E ' ' | - awk '{ print $1 }' | tr -d '<>' | sort -u || true -} +# The sync of an app made of what the lib covers: the namespace ontology in admin/model/ns.ttl if the +# model changed, the document tree, the documents whose sources are gone, and the rows of imports.csv +# whose files changed. An app with other steps calls the parts itself. +ldh_app_sync() { + local app_dir="$1" -# Runs the import command "${@:2}" into document $1 and records the import it creates in -# LDH_APP_IMPORTS, for ldh_app_await_imports. -LDH_APP_IMPORTS=() -ldh_app_track_import() { - local doc="$1" before uri - shift - before=$(ldh_app_import_uris "$doc") - "$@" - while IFS= read -r uri; do - [ -n "$uri" ] && LDH_APP_IMPORTS+=("$uri") - done < <(comm -13 <(echo "$before") <(ldh_app_import_uris "$doc")) -} + if [ -f "$app_dir/admin/model/ns.ttl" ] && ldh_app_changed 'admin/model/*'; then + ldh_app_step "Importing namespace ontology" + ldh_app_import_ns "$app_dir/admin/model" + fi -# Waits until every import in LDH_APP_IMPORTS carries prov:endedAtTime, which the platform adds to the -# import's own document when it finishes. Fails when one of them reports an error (a prov:Entity with -# a dct:description) or when LDH_IMPORT_TIMEOUT seconds (default 1200) pass first. -ldh_app_await_imports() { - [ "${#LDH_APP_IMPORTS[@]}" -gt 0 ] || return 0 - local deadline=$(( $(date +%s) + ${LDH_IMPORT_TIMEOUT:-1200} )) - local uri doc triples errors failed=0 - for uri in "${LDH_APP_IMPORTS[@]}"; do - doc="${uri%%#*}" - printf "Waiting for %s\n" "$uri" - while :; do - triples=$(ldh get --accept application/n-triples "$doc") - grep -qF "<$uri> " <<< "$triples" && break - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "Timed out waiting for $uri" >&2 - return 1 - fi - sleep 10 - done - # the error entities point at the import with prov:wasGeneratedBy - errors=$(grep -F " <$uri> " <<< "$triples" | awk '{ print $1 }' | - while IFS= read -r entity; do - grep -F "$entity " <<< "$triples" || true - done) - if [ -n "$errors" ]; then - echo "Import failed: $uri" >&2 - echo "$errors" >&2 - failed=1 - fi - done - return "$failed" + ldh_app_step "Pushing changed documents and files" + ldh_app_push_changed "$app_dir" + + ldh_app_step "Deleting removed documents" + ldh_app_delete_removed "$app_dir" + + if [ -f "$app_dir/imports.csv" ]; then + ldh_app_step "Importing changed CSV data" + ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" + fi } diff --git a/packages/sync.sh b/packages/sync.sh index e19aea58..014c2fc6 100755 --- a/packages/sync.sh +++ b/packages/sync.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Brings the package registry on a LinkedDataHub instance up to date with this folder after a commit: # pushes the package descriptors and files if one changed, deletes the documents whose sources are -# gone and re-declares the stylesheet of every package whose stylesheet changed. Dataspaces that -# already import a package keep the stylesheet they copied until the package is added again. +# gone, and declares the package stylesheets again whenever the push ran, since its PUTs replace every +# descriptor and with it the ac:stylesheet install.sh added. Dataspaces that already import a package +# keep the stylesheet they copied until the package is added again. # # Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). set -euo pipefail @@ -11,15 +12,13 @@ app_dir="$(cd "$(dirname "$0")" && pwd)" . "$app_dir/../lib/ldh-app.sh" ldh_app_require_env -ldh_app_step "Pushing changed package descriptors and files" -ldh_app_push_changed "$app_dir" +ldh_app_sync "$app_dir" -ldh_app_step "Deleting removed documents" -ldh_app_delete_removed "$app_dir" +[ "$LDH_APP_PUSHED" = 1 ] || exit 0 -ldh_app_step "Declaring changed package stylesheets" +ldh_app_step "Declaring package stylesheets" +# a package is the folder its stylesheet is in, e.g. editor/taxonomy/skos.xsl -> ${LDH_BASE}editor/taxonomy/#this (cd "$app_dir" && find . -name '*.xsl' -not -path './.*' | sed 's|^\./||' | sort) | while read -r stylesheet; do - ldh_app_changed "$stylesheet" || continue package_doc="${LDH_BASE}$(dirname "$stylesheet")/" upload="${LDH_BASE}uploads/$(shasum -a 1 "$app_dir/$stylesheet" | cut -d' ' -f1)" printf "Declaring %s as the stylesheet of %s#this\n" "$upload" "$package_doc"