From ce37644f5403d6b34f96810ef903056b57ac2bd4 Mon Sep 17 00:00:00 2001 From: kzoeps Date: Fri, 13 Nov 2026 15:01:47 +0600 Subject: [PATCH] ci: parallelize resilience profiles, worker-takeover and database suites - Run each deployment profile as its own matrix job; fan-in job runs assert-resilience-profiles on the uploaded receipts. - Run worker-takeover beside journey and cross-node-signin (default 120s lease kept). - Run the SQLite and PostgreSQL contract suites concurrently; the PostgreSQL restart/restore profile reuses the databases job sandbox. --- .github/workflows/ci.yml | 82 +++++++++++++++++++------ docs/testing.md | 13 +++- tests/local.sh | 26 ++++++-- tests/support/run-postgresql-profile.sh | 30 +++++++-- tests/support/run-resilience-profile.sh | 12 +++- 5 files changed, 131 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5ccecb9..5379df5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,12 +156,18 @@ jobs: run: | bash tests/local.sh prepare - name: Build and start managed application and test images + env: + DATABASE_BACKEND: postgresql + DEPLOYMENT_MODE: single-node + DATABASE_URL: postgresql://authority_owner@entryway-postgres/account_authority run: | bash tests/local.sh up - name: SQLite and PostgreSQL contracts run: | bash tests/local.sh database-dialects - name: PostgreSQL login restart and restore + env: + POSTGRESQL_PROFILE_REUSE_SANDBOX: "1" run: | bash tests/support/run-postgresql-profile.sh - name: Clean only recorded projects and children @@ -171,14 +177,20 @@ jobs: python3 scripts/ci-ownership.py cleanup "$CI_OWNERSHIP_MANIFEST" fi profiles: + name: profiles (${{ matrix.profile }}) runs-on: ubuntu-24.04 timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + profile: + [sqlite-single-node, postgresql-single-node, postgresql-multi-node] env: SOURCE_CHECKOUT: ${{ github.workspace }} - SANDBOX_PROJECT: hypercerts-entryway-ci-profiles-${{ github.run_id }}-${{ github.run_attempt }} - SANDBOX_CHECKOUT: ${{ github.workspace }}/tests/.runtime/ci-profiles-${{ github.run_id }}-${{ github.run_attempt }} - ACCEPTANCE_REPORT_DIR: ${{ github.workspace }}/tests/artifacts/ci-profiles-${{ github.run_id }}-${{ github.run_attempt }}/reports - CI_OWNERSHIP_MANIFEST: ${{ github.workspace }}/tests/artifacts/ci-profiles-${{ github.run_id }}-${{ github.run_attempt }}/ownership.json + SANDBOX_PROJECT: hypercerts-entryway-ci-profile-${{ matrix.profile }}-${{ github.run_id }}-${{ github.run_attempt }} + SANDBOX_CHECKOUT: ${{ github.workspace }}/tests/.runtime/ci-profile-${{ matrix.profile }}-${{ github.run_id }}-${{ github.run_attempt }} + ACCEPTANCE_REPORT_DIR: ${{ github.workspace }}/tests/artifacts/ci-profile-${{ matrix.profile }}-${{ github.run_id }}-${{ github.run_attempt }}/reports + CI_OWNERSHIP_MANIFEST: ${{ github.workspace }}/tests/artifacts/ci-profile-${{ matrix.profile }}-${{ github.run_id }}-${{ github.run_attempt }}/ownership.json steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -202,25 +214,59 @@ jobs: mkdir -p "$SANDBOX_CHECKOUT" "$ACCEPTANCE_REPORT_DIR" - name: Initialize deployment profile collection run: | - echo "PROFILE_COLLECTION_INDEX=$ACCEPTANCE_REPORT_DIR/profiles.tsv" >> "$GITHUB_ENV" + echo "PROFILE_COLLECTION_INDEX=$ACCEPTANCE_REPORT_DIR/profiles-${{ matrix.profile }}.tsv" >> "$GITHUB_ENV" echo "KEEP_PROFILE_SANDBOX=1" >> "$GITHUB_ENV" - - name: sqlite-single-node + - name: ${{ matrix.profile }} run: | - bash tests/support/run-resilience-profile.sh sqlite-single-node - - name: postgresql-single-node - run: | - bash tests/support/run-resilience-profile.sh postgresql-single-node - - name: postgresql-multi-node - run: | - bash tests/support/run-resilience-profile.sh postgresql-multi-node - - name: Assert all deployment profile receipts - run: | - last=$(tail -1 "$PROFILE_COLLECTION_INDEX" | cut -f2) - mapfile -t identity < "$ACCEPTANCE_REPORT_DIR/$last/sandbox-identity.txt" - SANDBOX_PROJECT="${identity[1]}" docker --context "$DOCKER_CONTEXT" compose --project-name "${identity[1]}" -f "${identity[0]}/compose.yaml" run --rm --no-deps -v "$ACCEPTANCE_REPORT_DIR:/profiles:ro" -v "$ACCEPTANCE_REPORT_DIR:/app/artifacts:z" test node tests/support/assert-resilience-profiles.mjs + bash tests/support/run-resilience-profile.sh "${{ matrix.profile }}" + # Only the files assert-resilience-profiles.mjs reads. Logs, commands and + # browser output can contain mailbox codes or sandbox identity. + - name: Upload deployment profile receipt + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: resilience-profile-${{ matrix.profile }} + path: | + ${{ env.ACCEPTANCE_REPORT_DIR }}/profiles-${{ matrix.profile }}.tsv + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/prepare.exit + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/up.exit + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/exercise.exit + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/assert-profile.exit + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/profile.exit + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/resilience-profile.json + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/profile-duration.json + ${{ env.ACCEPTANCE_REPORT_DIR }}/resilience-${{ matrix.profile }}-*/exercise-controller.json + if-no-files-found: warn + retention-days: 7 - name: Clean only recorded projects and children if: always() run: | if test -f "$CI_OWNERSHIP_MANIFEST"; then python3 scripts/ci-ownership.py cleanup "$CI_OWNERSHIP_MANIFEST" fi + profiles-assert: + name: profiles (assert all receipts) + needs: profiles + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: resilience-profile-* + merge-multiple: true + path: ${{ runner.temp }}/profiles + - name: Combine profile collection index + run: | + cat "$RUNNER_TEMP"/profiles/profiles-*.tsv > "$RUNNER_TEMP/profiles/profiles.tsv" + mkdir -p "$RUNNER_TEMP/assert-output" + # The receipts record a digest of the candidate files inside the test image. + # Build the same image from the same Dockerfile and context to recompute it. + - name: Build candidate test image + run: | + docker --context default build -f tests/support/Dockerfile -t entryway-profile-assert:local . + - name: Assert all deployment profile receipts + run: | + docker --context default run --rm -v "$RUNNER_TEMP/profiles:/profiles:ro" -v "$RUNNER_TEMP/assert-output:/app/artifacts:z" entryway-profile-assert:local node tests/support/assert-resilience-profiles.mjs diff --git a/docs/testing.md b/docs/testing.md index cfd32fdd..23edac72 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -173,9 +173,16 @@ Formatting, lint, syntax, types, architecture, schema, build, unit and lifecycle checks each have a visible step. Integration runs browser before contracts, then migration, the declared moderation-gap assertion, recovery/crash/authority checks and resilience last. Database dialects and the PostgreSQL application -profile are separate steps. Each deployment profile has its own step followed -by the aggregate receipt assertion. These gates do not close the pending public -migration or moderation requirements. +profile are separate steps: the databases job starts its sandbox on PostgreSQL, +runs the SQLite and PostgreSQL suites concurrently, and runs the application +profile in that sandbox (`POSTGRESQL_PROFILE_REUSE_SANDBOX=1`, which leaves +cleanup to the job; without it the profile owns a fresh sandbox). Each +deployment profile runs as its own parallel matrix job, `profiles ()`. +It uploads only the receipt files the aggregate assertion reads, and the +`profiles (assert all receipts)` job downloads them and asserts all three. In +the multi-node profile `worker-takeover` runs beside `journey` and +`cross-node-signin`, before any fault step. These gates do not close the pending +public migration or moderation requirements. Contributors reproduce basic checks directly from the repository root: diff --git a/tests/local.sh b/tests/local.sh index e1922da4..1de6c00f 100755 --- a/tests/local.sh +++ b/tests/local.sh @@ -152,13 +152,31 @@ case "$command" in database-contracts|database-dialects) require_state compose up -d --wait entryway-postgres - for backend in sqlite postgresql; do + # The suites share no state: SQLite uses in-memory databases in its own + # container, PostgreSQL gives each test a private schema, and each writes + # distinct report files. Output goes to per-backend logs so the two runs + # cannot interleave; summaries print after both finish. + pids=() + backends=(sqlite postgresql) + for backend in "${backends[@]}"; do + ( + result=0 + compose run --rm --no-deps -e "CONTRACT_DATABASE_BACKEND=$backend" -e CONTRACT_DATABASE_URL=postgresql://authority_owner@entryway-postgres/account_authority test node tests/support/run-database-contracts.mjs > "$artifacts/database-$backend.log" 2>&1 || result=$? + printf '%s\n' "$result" > "$artifacts/database-$backend.exit" + exit "$result" + ) & + pids+=("$!") + done + failed=0 + for i in "${!backends[@]}"; do + backend=${backends[$i]} result=0 - compose run --rm --no-deps -e "CONTRACT_DATABASE_BACKEND=$backend" -e CONTRACT_DATABASE_URL=postgresql://authority_owner@entryway-postgres/account_authority test node tests/support/run-database-contracts.mjs 2>&1 | tee "$artifacts/database-$backend.log" | python3 "$root/scripts/ci-summary.py" || result=$? - printf '%s\n' "$result" > "$artifacts/database-$backend.exit" + wait "${pids[$i]}" || result=$? + python3 "$root/scripts/ci-summary.py" < "$artifacts/database-$backend.log" || true echo "Database $backend: exit $result" - (( result == 0 )) || exit "$result" + (( result == 0 )) || failed=$result done + (( failed == 0 )) || exit "$failed" if [[ "$command" == database-contracts ]]; then bash "$root/tests/support/run-postgresql-profile.sh"; fi ;; resilience-profiles) diff --git a/tests/support/run-postgresql-profile.sh b/tests/support/run-postgresql-profile.sh index 1c910236..40e7746b 100755 --- a/tests/support/run-postgresql-profile.sh +++ b/tests/support/run-postgresql-profile.sh @@ -1,16 +1,26 @@ #!/usr/bin/env bash # Own a separate fresh AiaB project; private config/session/backup remain in volumes. +# With POSTGRESQL_PROFILE_REUSE_SANDBOX=1 (CI databases job) run in the caller's +# already prepared sandbox instead: nothing is prepared, registered or torn down +# here, so the caller must own cleanup. Set it only with SANDBOX_CHECKOUT and +# SANDBOX_PROJECT of that sandbox. set -euo pipefail root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) parent_artifacts=${ACCEPTANCE_REPORT_DIR:-"$root/tests/artifacts"} run="$(date +%s)-$$" -export SANDBOX_CHECKOUT="$root/tests/.runtime/postgresql-profile-$run" -export SANDBOX_PROJECT="hypercerts-entryway-postgresql-profile-$run" +reuse_sandbox=${POSTGRESQL_PROFILE_REUSE_SANDBOX:-0} +if [[ "$reuse_sandbox" == 1 ]]; then + : "${SANDBOX_CHECKOUT:?POSTGRESQL_PROFILE_REUSE_SANDBOX=1 needs the SANDBOX_CHECKOUT of a prepared sandbox}" "${SANDBOX_PROJECT:?POSTGRESQL_PROFILE_REUSE_SANDBOX=1 needs the SANDBOX_PROJECT of a prepared sandbox}" + [[ -f "$SANDBOX_CHECKOUT/compose.yaml" ]] || { echo "POSTGRESQL_PROFILE_REUSE_SANDBOX=1 found no $SANDBOX_CHECKOUT/compose.yaml; run tests/local.sh prepare and up first, or unset it for a standalone profile" >&2; exit 2; } +else + export SANDBOX_CHECKOUT="$root/tests/.runtime/postgresql-profile-$run" + export SANDBOX_PROJECT="hypercerts-entryway-postgresql-profile-$run" +fi export ACCEPTANCE_REPORT_DIR="$parent_artifacts/postgresql-profile-$run" export DATABASE_BACKEND=postgresql DEPLOYMENT_MODE=single-node export DATABASE_URL=postgresql://authority_owner@entryway-postgres/account_authority mkdir -p "$ACCEPTANCE_REPORT_DIR" -if [[ -n "${CI_OWNERSHIP_MANIFEST:-}" ]]; then +if [[ "$reuse_sandbox" != 1 && -n "${CI_OWNERSHIP_MANIFEST:-}" ]]; then python3 "$root/scripts/ci-ownership.py" register "$CI_OWNERSHIP_MANIFEST" "$SANDBOX_CHECKOUT" "$SANDBOX_PROJECT" fi source "$root/tests/support/rootless-docker.sh" @@ -19,11 +29,19 @@ compose() { docker --context "$DOCKER_CONTEXT" compose --project-name "$SANDBOX_ cleanup() { status=$? printf '%s\n' "$status" > "$ACCEPTANCE_REPORT_DIR/profile.exit" - if (( status == 0 )); then compose down --volumes --remove-orphans; else echo "Preserved owned PostgreSQL profile $SANDBOX_PROJECT" >&2; fi + if [[ "$reuse_sandbox" == 1 ]]; then echo "Left reused sandbox $SANDBOX_PROJECT to its owner" >&2 + elif (( status == 0 )); then compose down --volumes --remove-orphans; else echo "Preserved owned PostgreSQL profile $SANDBOX_PROJECT" >&2; fi } trap cleanup EXIT -"$root/tests/local.sh" prepare 2>&1 | tee "$ACCEPTANCE_REPORT_DIR/prepare.log" -"$root/tests/local.sh" up 2>&1 | tee "$ACCEPTANCE_REPORT_DIR/up.log" +if [[ "$reuse_sandbox" == 1 ]]; then + # No-op when the sandbox already runs on PostgreSQL; otherwise recreates only + # entryway against it. A stale SQLite file from a default start is then + # reported by the profile's no-SQLite-fallback check. + compose up -d --wait entryway-postgres entryway 2>&1 | tee "$ACCEPTANCE_REPORT_DIR/up.log" +else + "$root/tests/local.sh" prepare 2>&1 | tee "$ACCEPTANCE_REPORT_DIR/prepare.log" + "$root/tests/local.sh" up 2>&1 | tee "$ACCEPTANCE_REPORT_DIR/up.log" +fi probe() { local phase="$1" echo "PostgreSQL profile: $phase" diff --git a/tests/support/run-resilience-profile.sh b/tests/support/run-resilience-profile.sh index baad22ee..612a03bc 100644 --- a/tests/support/run-resilience-profile.sh +++ b/tests/support/run-resilience-profile.sh @@ -98,10 +98,20 @@ for node in entryway $([[ "$ENTRYWAY_PROFILE_NODE_COUNT" == 2 ]] && echo entrywa done sha256sum "$SANDBOX_CHECKOUT/compose.yaml" "$SANDBOX_CHECKOUT/state/manifest.json" > "$ACCEPTANCE_REPORT_DIR/generated-inputs.sha256" run readiness-initial probe ready readiness-initial.json +if [[ "$ENTRYWAY_PROFILE_NODE_COUNT" == 2 ]]; then + # worker-takeover waits for the real 120s account lease. It uses a private + # PostgreSQL schema and controlled transport, so it shares no rows, accounts or + # workers with the live nodes. Start it before any fault step and wait for it + # before the first one, so it overlaps only journey and cross-node-signin. + run worker-takeover phase_container -e CONTRACT_DATABASE_BACKEND=postgresql -e CONTRACT_DATABASE_URL=postgresql://authority_owner@entryway-postgres/account_authority test timeout --signal=TERM --kill-after=5s 195s node --test tests/contracts/profile-workers.mjs & + takeover_pid=$! +fi run journey browser journey journey if [[ "$ENTRYWAY_PROFILE_NODE_COUNT" == 2 ]]; then run cross-node-signin browser cross-node-signin cross-node-signin - run worker-takeover phase_container -e CONTRACT_DATABASE_BACKEND=postgresql -e CONTRACT_DATABASE_URL=postgresql://authority_owner@entryway-postgres/account_authority test timeout --signal=TERM --kill-after=5s 195s node --test tests/contracts/profile-workers.mjs + takeover_status=0 + wait "$takeover_pid" || takeover_status=$? + (( takeover_status == 0 )) || { echo "Profile $profile: worker-takeover failed (exit $takeover_status)" >&2; exit "$takeover_status"; } fi # Graceful stop is bounded separately from the application deadline. Preserve the # real container exit and test existing sessions/direct writes on the survivor.