diff --git a/CLAUDE.md b/CLAUDE.md index 03567b6..d8f95f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ LinkedDataHub-Apps holds dataspaces built on LinkedDataHub. Every dataspace is a - `demo/copenhagen/` — Copenhagen open data on a map, imported from CSV - `demo/unesco-thesaurus/` — the UNESCO Thesaurus as a SKOS editor; its model and stylesheet come from the taxonomy editor package - `packages/` — the package registry published at https://packages.linkeddatahub.com/; one package so far, `editor/taxonomy` -- `lib/` — the shared installer: `install.mk` (the interactive `make install`) and `ldh-app.sh` (functions every `install.sh` sources) +- `lib/` — the shared installer: `install.mk` (the interactive `make install` and `make sync`) and `ldh-app.sh` (functions every `install.sh` and `sync.sh` sources) - `screencast/` — the Playwright rig that produces the docs' screenshots and clips, and the Northwind screencast tracks - `check-links.sh`, `check-xmlliterals.sh` — repository-wide validators, run by CI @@ -43,6 +43,10 @@ Each `install.sh` sources `lib/ldh-app.sh` and is the app's own sequence of step **Re-runs converge but are not clean.** `ldh push` PUTs and the ontology is reset before re-import, so documents and the model end up the same. `make-public`, `create authorization` and every import are POSTs, so each run adds another authorization and another import record. Nothing probes for an earlier install. +### Syncing an app + +`sync.sh`, beside each `install.sh`, brings an installed app up to date and is safe to run on every commit. It skips the one-off steps (authorizations, `packages add`), re-imports the namespace ontology only when `admin/model/` changed, pushes only when a file in the push plan changed (`ldh push --dry-run`, so `.ldhignore` holds), re-runs only the import rows whose query or CSV changed, each with `--wait`, and deletes the documents whose sources are gone. Two optional variables describe the commit: `LDH_CHANGED`, the added or modified paths relative to the app folder, and `LDH_PREVIOUS_TREE`, the app folder as it was before. With neither, `make sync` is a full sync. Mapping commit ranges onto apps is the deployment's job; linkeddatahub.com does it in its own repository. + ### App layout The file tree is the URL tree. `categories.ttl` installs to `{base}categories/`, the sibling folder `categories/` holds that document's uploads (and, by convention, its `.csv` and mapping `.rq`), and `root.ttl` is the base document itself. `.ldhignore` lists what is neither a document nor an upload — `admin/`, `Makefile`, `*.sh`, `*.csv`, repository screenshots — because the CLI skips nothing but hidden entries. diff --git a/demo/copenhagen/sync.sh b/demo/copenhagen/sync.sh new file mode 100755 index 0000000..5cc398f --- /dev/null +++ b/demo/copenhagen/sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports +# the namespace ontology if the model changed, pushes the document tree if a pushed document or file +# changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or +# data changed, waiting for each to end. Safe to run on every commit; install.sh does the one-off +# steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_sync "$app_dir" diff --git a/demo/northwind-traders/sync.sh b/demo/northwind-traders/sync.sh new file mode 100755 index 0000000..5cc398f --- /dev/null +++ b/demo/northwind-traders/sync.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: re-imports +# the namespace ontology if the model changed, pushes the document tree if a pushed document or file +# changed, deletes the documents whose sources are gone, and re-runs the CSV imports whose query or +# data changed, waiting for each to end. Safe to run on every commit; install.sh does the one-off +# steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_sync "$app_dir" diff --git a/demo/unesco-thesaurus/sync.sh b/demo/unesco-thesaurus/sync.sh new file mode 100755 index 0000000..2cb0790 --- /dev/null +++ b/demo/unesco-thesaurus/sync.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Brings the app on a LinkedDataHub instance up to date with this folder after a commit: pushes the +# document tree if a pushed document or file changed, deletes the documents whose sources are gone, +# and re-imports the UNESCO Thesaurus if its data or query changed, waiting for it to end. Safe to run +# on every commit; install.sh does the one-off steps, the package import among them. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_sync "$app_dir" + +if ldh_app_changed 'concept-schemes/unesco-thesaurus/*'; then + ldh_app_step "Importing SKOS vocabulary" + ldh import rdf \ + --title "Unesco Thesaurus SKOS" \ + --query-file "$app_dir/concept-schemes/unesco-thesaurus/skos-import.rq" \ + --rdf-file "$app_dir/concept-schemes/unesco-thesaurus/unesco-thesaurus.ttl" \ + --content-type "text/turtle" \ + --wait \ + "${LDH_BASE}concept-schemes/unesco-thesaurus/" < /dev/null +fi diff --git a/docs/sync.sh b/docs/sync.sh new file mode 100755 index 0000000..19f486f --- /dev/null +++ b/docs/sync.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Brings the documentation on a LinkedDataHub instance up to date with this folder after a commit: +# pushes the document tree if a pushed document or file changed and deletes the documents whose +# sources are gone. Safe to run on every commit; install.sh does the one-off steps. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_sync "$app_dir" diff --git a/lib/install.mk b/lib/install.mk index 363ce0d..bf77455 100644 --- a/lib/install.mk +++ b/lib/install.mk @@ -1,14 +1,14 @@ # Shared by every app Makefile. `make install` prompts for the dataspace and the owner's certificate, # exports them as the LDH_* variables the ldh CLI reads, and runs the app's install.sh in that -# environment. No value travels on a command line, so the password needs no escaping and is not -# visible in the process list. +# environment; `make sync` does the same for sync.sh, a full sync. No value travels on a command line, +# so the password needs no escaping and is not visible in the process list. # # An app Makefile sets the defaults it wants to offer, then includes this file: # # CERT_PATH_DEFAULT = ../../../LinkedDataHub/ssl/owner/keystore.p12 # include ../../lib/install.mk # -# To install unattended, export the variables and run ./install.sh yourself. +# To install or sync unattended, export the variables and run ./install.sh or ./sync.sh yourself. SHELL := /bin/bash @@ -16,9 +16,9 @@ BASE_URL_DEFAULT ?= https://localhost:4443/ CERT_PATH_DEFAULT ?= ../../LinkedDataHub/ssl/owner/keystore.p12 PROXY_URL_DEFAULT ?= https://localhost:5443/ -.PHONY: install +.PHONY: install sync -install: +install sync: @read -p "Enter Base URL [$(BASE_URL_DEFAULT)]: " BASE_URL; \ read -p "Enter Certificate Path [$(CERT_PATH_DEFAULT)]: " CERT_PATH; \ read -s -p "Enter Certificate Password (required): " PASSWORD; \ @@ -32,4 +32,4 @@ install: export LDH_CERT_FILE="$${CERT_PATH:-$(CERT_PATH_DEFAULT)}"; \ export LDH_CERT_PASSWORD="$${PASSWORD}"; \ if [ -n "$${PROXY_URL}" ]; then export LDH_PROXY="$${PROXY_URL}"; fi; \ - ./install.sh + ./$@.sh diff --git a/lib/ldh-app.sh b/lib/ldh-app.sh index 92f2f62..7bc0d13 100755 --- a/lib/ldh-app.sh +++ b/lib/ldh-app.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Shared by every install.sh in this repository. Sourced, not run. +# Shared by every install.sh and sync.sh in this repository. Sourced, not run. # # The ldh CLI reads its connection from the environment — LDH_BASE, LDH_CERT_FILE, LDH_CERT_PASSWORD # and LDH_PROXY — so the installers put no credential on any command line: nothing to escape, nothing @@ -37,12 +37,14 @@ ldh_app_import_ns() { # Runs one ldh import csv per row of the manifest $2 — query_filename,csv_filename,target,title, paths # relative to $1: it adds the CONSTRUCT query, uploads the CSV into the target document and creates # the import that maps one into the other, then waits for the import to end and fails if it did. Every -# run creates a new import; the documents the imports write are the same, so the data converges. +# run creates a new import; the documents the imports write are the same, so the data converges. A sync +# runs only the rows whose query or CSV changed. ldh_app_import_csv() { local app_dir="$1" manifest="$2" local query_filename csv_filename target_path title while IFS=, read -r query_filename csv_filename target_path title; do [ -n "$query_filename" ] || continue + ldh_app_changed "$query_filename" "$csv_filename" || continue printf "Importing %s into %s\n" "$csv_filename" "${LDH_BASE}${target_path}" ldh import csv \ --title "$title" \ @@ -52,3 +54,93 @@ ldh_app_import_csv() { "${LDH_BASE}${target_path}" < /dev/null done < <(tail -n +2 "$manifest") } + +# --- sync ------------------------------------------------------------------------------------------- +# +# A sync.sh redoes only what the changed files require, so it can run on every commit; install.sh +# stays the one-off (authorizations, packages). Two variables describe the commit, both optional: +# +# LDH_CHANGED newline-separated paths, relative to the app folder, added or modified by it. +# Unset means everything changed: `make sync` with neither set is a full sync. +# LDH_PREVIOUS_TREE the app folder as it was before the commit. When set, documents its files +# mapped to and the current files no longer do are deleted. + +# True when any of the patterns (globs relative to the app folder, * crossing slashes) matches a +# changed path, and always when LDH_CHANGED is unset. +ldh_app_changed() { + [ -n "${LDH_CHANGED+set}" ] || return 0 + local path pattern + while IFS= read -r path; do + [ -n "$path" ] || continue + for pattern in "$@"; do + # shellcheck disable=SC2053 # the pattern is a glob on purpose + [[ "$path" == $pattern ]] && return 0 + done + done <<< "$LDH_CHANGED" + return 1 +} + +# The plan `ldh push` makes for folder $1 onto $LDH_BASE, one "METHOD URI <- relative path" line per +# step. A dry run reads the files and sends nothing. +ldh_app_push_plan() { + ldh push --dry-run --dir "$1" "$LDH_BASE" 2>&1 >/dev/null | grep -E '^(PUT|POST) ' || true +} + +# Pushes app folder $1 when a changed file is part of its push plan: a document it PUTs or a file it +# uploads. Files the push ignores (.ldhignore) never trigger it. The push PUTs every document, not only +# the changed ones, so LDH_APP_PUSHED is set to 1 for whatever a sync must add back afterwards. +LDH_APP_PUSHED=0 +ldh_app_push_changed() { + local app_dir="$1" method uri arrow relative + if [ -n "${LDH_CHANGED+set}" ]; then + while read -r method uri arrow relative; do + ldh_app_changed "$relative" && break + relative="" + done < <(ldh_app_push_plan "$app_dir") + if [ -z "$relative" ]; then + echo "No pushed document or file changed" + return 0 + fi + fi + ldh push --dir "$app_dir" "$LDH_BASE" < /dev/null + LDH_APP_PUSHED=1 +} + +# Deletes the documents app folder $1 no longer maps to: those in the push plan of LDH_PREVIOUS_TREE +# that are not in the current one, children before their parents. Uploads are kept, since other +# documents may still link them, and so are documents an import wrote, which no plan contains. +ldh_app_delete_removed() { + local app_dir="$1" uri + [ -n "${LDH_PREVIOUS_TREE:-}" ] && [ -d "$LDH_PREVIOUS_TREE" ] || return 0 + while IFS= read -r uri; do + [ -n "$uri" ] || continue + printf "Deleting %s\n" "$uri" + ldh delete "$uri" < /dev/null + done < <(comm -23 \ + <(ldh_app_push_plan "$LDH_PREVIOUS_TREE" | awk '$1 == "PUT" { print $2 }' | sort -u) \ + <(ldh_app_push_plan "$app_dir" | awk '$1 == "PUT" { print $2 }' | sort -u) \ + | sort -r) +} + +# The sync of an app made of what the lib covers: the namespace ontology in admin/model/ns.ttl if the +# model changed, the document tree, the documents whose sources are gone, and the rows of imports.csv +# whose files changed. An app with other steps calls the parts itself. +ldh_app_sync() { + local app_dir="$1" + + if [ -f "$app_dir/admin/model/ns.ttl" ] && ldh_app_changed 'admin/model/*'; then + ldh_app_step "Importing namespace ontology" + ldh_app_import_ns "$app_dir/admin/model" + fi + + ldh_app_step "Pushing changed documents and files" + ldh_app_push_changed "$app_dir" + + ldh_app_step "Deleting removed documents" + ldh_app_delete_removed "$app_dir" + + if [ -f "$app_dir/imports.csv" ]; then + ldh_app_step "Importing changed CSV data" + ldh_app_import_csv "$app_dir" "$app_dir/imports.csv" + fi +} diff --git a/packages/sync.sh b/packages/sync.sh new file mode 100755 index 0000000..014c2fc --- /dev/null +++ b/packages/sync.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Brings the package registry on a LinkedDataHub instance up to date with this folder after a commit: +# pushes the package descriptors and files if one changed, deletes the documents whose sources are +# gone, and declares the package stylesheets again whenever the push ran, since its PUTs replace every +# descriptor and with it the ac:stylesheet install.sh added. Dataspaces that already import a package +# keep the stylesheet they copied until the package is added again. +# +# Reads the LDH_* connection like install.sh, and LDH_CHANGED / LDH_PREVIOUS_TREE (see lib/ldh-app.sh). +set -euo pipefail + +app_dir="$(cd "$(dirname "$0")" && pwd)" +. "$app_dir/../lib/ldh-app.sh" +ldh_app_require_env + +ldh_app_sync "$app_dir" + +[ "$LDH_APP_PUSHED" = 1 ] || exit 0 + +ldh_app_step "Declaring package stylesheets" +# a package is the folder its stylesheet is in, e.g. editor/taxonomy/skos.xsl -> ${LDH_BASE}editor/taxonomy/#this +(cd "$app_dir" && find . -name '*.xsl' -not -path './.*' | sed 's|^\./||' | sort) | while read -r stylesheet; do + package_doc="${LDH_BASE}$(dirname "$stylesheet")/" + upload="${LDH_BASE}uploads/$(shasum -a 1 "$app_dir/$stylesheet" | cut -d' ' -f1)" + printf "Declaring %s as the stylesheet of %s#this\n" "$upload" "$package_doc" + echo "INSERT { <${package_doc}#this> <${upload}> } WHERE { }" | + ldh patch "$package_doc" +done