Release 1.3.0-beta.0: --syncDeletes, linked content levels, list order sync, PROD-3042/3043 fixes

.github/workflows/npm-publish.yml

@@ -22,19 +22,17 @@ on:
         default: latest
 
 jobs:
-  publish:
-    name: Build and publish to npm
+  # A v* tag push: check the tag, run the sync permutation test on the tag's build, and publish
+  # only if it passed. Nothing reaches npm (or gets a dist-tag) untested on dev.
+  check-version:
+    name: Check the tag matches package.json
     if: github.event_name == 'push'
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      id-token: write
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
 
       - name: Verify tag matches package.json version
         run: |
@@ -45,6 +43,31 @@ jobs:
             exit 1
           fi
 
+  sync-test:
+    name: Sync permutation test
+    needs: check-version
+    uses: ./.github/workflows/sync-permutation.yml
+    with:
+      tier: beta
+    secrets: inherit
+    permissions:
+      contents: read
+      id-token: write
+
+  publish:
+    name: Build and publish to npm
+    needs: sync-test
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
@@ -68,8 +91,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      # Reads the sync permutation runs for the gate below.
-      actions: read
       # Uses the trusted publisher's OIDC credentials; needs "Allow npm dist-tag"
       # enabled on the package's trusted publishing configuration on npmjs.com.
       id-token: write
@@ -106,23 +127,9 @@ jobs:
       - name: Check the version is published
         run: npm view "@agility/cli@$VERSION" version | grep -qx "$VERSION" || { echo "@agility/cli@$VERSION is not on npm."; exit 1; }
 
-      # Nothing reaches `latest` untested on dev: the tag's sync permutation run
-      # (sync-permutation.yml, triggered by the same tag push) must have passed. If it failed
-      # for a reason unrelated to the CLI, re-run it from the Actions tab and promote again.
-      - name: Require a passing sync permutation run
-        if: env.DIST_TAG == 'latest'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          REPO: ${{ github.repository }}
-        run: |
-          SHA=$(gh api "repos/$REPO/commits/v$VERSION" --jq .sha) || { echo "No tag v$VERSION."; exit 1; }
-          PASSED=$(gh api "repos/$REPO/actions/workflows/sync-permutation.yml/runs?event=push&head_sha=$SHA&status=success&per_page=1" --jq .total_count)
-          if [ "$PASSED" = "0" ]; then
-            echo "No passing 'Sync permutation test' run for v$VERSION ($SHA). See the Actions tab."
-            exit 1
-          fi
-          echo "v$VERSION passed the sync permutation test."
-
+      # No separate test check: since the sync permutation test gates the publish, every version
+      # published from now on has passed it. (Versions published before that, 1.2.2 and earlier,
+      # never went through it.)
       - name: Promote
         run: npm dist-tag add "@agility/cli@$VERSION" "$DIST_TAG"
 

.github/workflows/sync-permutation.yml

@@ -12,19 +12,28 @@ name: Sync permutation test
 #     whatever happens to the rest of the run, and the leasing service deletes anything a
 #     dead runner leaves behind once the TTL runs out.
 #
-# Tiers (the same jobs; the tier sets TTLs and timeouts, and later slices add scenarios to it):
-#   quick    every pull request
-#   beta     every v* tag (the version npm-publish.yml just published under `beta`). Its result
-#            gates promotion to `latest` (npm-publish.yml, promote job)
-#   nightly  every night against whatever is deployed to dev, plus a sweep for left-behind leases
+# It leases two instances and runs for 10 to 15 minutes, so it runs only for a release and by hand:
+#   release  npm-publish.yml calls it on every v* tag push, on that tag's build (tier `beta`), and
+#            publishes to npm only if it passes
+#   by hand  Actions tab → Sync permutation test → Run workflow, on a branch or a published version
+#
+# Tiers (the same jobs; the tier sets TTLs, timeouts and the number of change rounds):
+#   quick    two fixed rounds
+#   beta     four seeded random rounds (a release)
+#   nightly  six seeded random rounds
 
 on:
-  pull_request:
-  push:
-    tags:
-      - "v*"
-  schedule:
-    - cron: "0 7 * * *" # 07:00 UTC, 03:00 Eastern
+  workflow_call:
+    inputs:
+      tier:
+        type: string
+        default: beta
+      cli_version:
+        type: string
+        default: ""
+      seed:
+        type: string
+        default: ""
   workflow_dispatch:
     inputs:
       tier:
@@ -42,28 +51,24 @@ on:
         default: ""
 
 concurrency:
-  group: sync-permutation-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  group: sync-permutation-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: false
 
 permissions:
   contents: read
   id-token: write
 
 env:
-  TIER: ${{ inputs.tier || (github.event_name == 'schedule' && 'nightly') || (github.ref_type == 'tag' && 'beta') || 'quick' }}
-  # Published version under test: the dispatch input, or the tag's version on a tag push.
-  CLI_VERSION: ${{ inputs.cli_version || (github.ref_type == 'tag' && github.ref_name) || '' }}
+  TIER: ${{ inputs.tier || 'quick' }}
+  # Blank: build and test this checkout (a release tests its tag's build). Otherwise a published version.
+  CLI_VERSION: ${{ inputs.cli_version || '' }}
   # Every lease of this run (and of this attempt, so a re-run gets fresh instances) starts with this.
   LEASE_PREFIX: cli-perm-${{ github.run_id }}-${{ github.run_attempt }}-
   GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
 jobs:
   lease:
     name: Lease dev instances
-    # Same-repo PRs only: forks and Dependabot get no credentials.
-    if: >-
-      github.event_name != 'pull_request' ||
-      (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')
     runs-on: ubuntu-latest
     environment: cli-dev-live
     timeout-minutes: 20
@@ -94,7 +99,6 @@ jobs:
 
       # Layer 4 of teardown: anything an earlier run left behind (its release job failed or never ran).
       - name: Sweep left-behind leases
-        if: env.TIER == 'nightly'
         run: node ci/leases.mjs sweep cli-perm- --older-than-hours 6
 
       # TTL = the run's budget plus a margin, so the reaper is the backstop if every other layer fails.
@@ -109,7 +113,7 @@ jobs:
     needs: lease
     runs-on: ubuntu-latest
     environment: cli-dev-live
-    timeout-minutes: ${{ (github.event_name == 'schedule' || inputs.tier == 'nightly') && 180 || 120 }}
+    timeout-minutes: ${{ inputs.tier == 'nightly' && 180 || 120 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -133,11 +137,6 @@ jobs:
         if: env.CLI_VERSION != ''
         run: |
           V="${CLI_VERSION#v}"
-          # A tag push starts this workflow and npm-publish.yml together; wait for the publish.
-          for i in $(seq 1 40); do
-            npm view "@agility/cli@$V" version >/dev/null 2>&1 && break
-            echo "waiting for @agility/cli@$V on npm ($i/40)"; sleep 15
-          done
           npm install --prefix "$RUNNER_TEMP/cli" "@agility/cli@$V"
           echo "CLI=node $RUNNER_TEMP/cli/node_modules/@agility/cli/dist/index.js" >> "$GITHUB_ENV"
 
@@ -178,11 +177,7 @@ jobs:
   release:
     name: Release leases
     needs: [lease, test]
-    if: >-
-      always() && (
-        github.event_name != 'pull_request' ||
-        (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')
-      )
+    if: always()
     runs-on: ubuntu-latest
     environment: cli-dev-live
     timeout-minutes: 10

README.md

@@ -40,6 +40,20 @@ agility pull [options]
 | `--elements` | string | `Models,Galleries,Assets,Containers,Content,Templates,Pages,Sitemaps,UrlRedirections` | Comma-separated list of elements to process                                             |
 | `--models`   | string | _(empty)_                                                            | Comma-separated list of model reference names to sync (only syncs the specified models) |
 
+#### Syncing deletes
+
+`--syncDeletes` adds a final phase to a sync. For every mapping whose source entity is gone from the source pull it decides:
+
+| Outcome | When |
+| --- | --- |
+| **deleted** | The source entity is deleted and the target is unchanged since the last sync (or `--overwrite` is set). The mapping is removed. |
+| **conflict** (kept) | The target was edited since the last sync. Re-run with `--overwrite` to delete it anyway. |
+| **blocked** (kept) | Something that stays on the target still uses it — e.g. a target-only child page, a page or item that links to the content item, items in the container, a container or item built on the model, content that embeds the asset. |
+| **mapping removed** | The target is already gone, or another live source entity maps to the same target. |
+| **kept** | The source item is unpublished rather than deleted, or the delete could not be confirmed. |
+
+Run it with `--preflight` first: the report lists every planned delete under `Delete …` phases. Deleted pages and content items go to the instance's normal delete flow (the batch is labelled "Deleted by agility-cli --syncDeletes").
+
 **Authentication & Environment Options:**
 
 | Option    | Type    | Default | Description                                                                                                                              |
@@ -105,6 +119,8 @@ agility sync [options]
 | Option          | Type    | Default      | Description                                                                                                                                                                                                                                           |
 | --------------- | ------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `--overwrite`   | boolean | `false`      | Conflict-scoped override. By default, when a target item has its own changes that conflict with the source, the CLI skips it to avoid data loss. With `--overwrite`, those conflicting target items are overwritten with the source version. Non-conflicting updates are applied either way. Also covers a mapped target item that no longer exists (deleted or unpublished on the target): by default that is reported as a conflict and skipped, with `--overwrite` the item is recreated and the mapping repointed. |
+| `--syncDeletes` | boolean | `false`      | After syncing, delete from the target the entities that were deleted in the source since the last sync (URL redirections, pages, content, containers, templates, models, assets, galleries — dependents first). Only entities the CLI previously synced (that have a mapping) are considered; nothing created directly on the target is touched. A target entity edited since the last sync is reported as a conflict and kept unless `--overwrite` is also set. An entity that something kept on the target still uses (a child page, a page or item that links to the content, a container's items, ...) is kept and reported. Content and page deletes are confirmed against the source first, so an item that was only unpublished is never deleted. Cannot be combined with `--pages`, `--containers`, `--models` or `--models-with-deps`; use `--elements` to limit the types. Works with `--preflight`. Not read from `.env`. See [Syncing deletes](#syncing-deletes). |
+| `--allowMassDeletes` | boolean | `false` | With `--syncDeletes`: allow a run to delete more than half (and more than 10) of the mapped entities of one type. Without it such a run deletes nothing and fails, because a stale or partial local cache looks the same as a mass delete. |
 | `--autoPublish` | string  | _(disabled)_ | Automatically publish synced items that were published in the source instance. Values: `content`, `pages`, `both`. If flag is provided without a value, defaults to `both`. Items that are only in staging (not published) in the source are skipped. |
 | `--fullPull`    | boolean | `false`      | Full content re-pull. Discards the stored content sync token for every locale, re-pulls all content items and pages, and removes local content/page files that no longer exist on the instance. Use when a local cache is suspected stale. Models, containers, templates, galleries and assets reconcile against the instance on every pull regardless. |
 | `--preflight`   | boolean | `false`      | Dry-run preview. Runs the full source-pull, target-pull, dependency analysis and change detection, then reports the creates/updates/skips/conflicts a real sync would produce — **without writing anything** to the target instance or mapping files. Exits non-zero when conflicts are detected, so CI can gate a real sync on a clean preflight. |
@@ -141,6 +157,10 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --overwrite
 # Preview what a sync would do without writing anything (exits non-zero if conflicts are found)
 agility sync --sourceGuid="abc123" --targetGuid="def456" --preflight
 
+# Also delete from the target what was deleted in the source since the last sync (preview first)
+agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes --preflight
+agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes
+
 # Sync only specified models (models only, no dependencies)
 agility sync --sourceGuid="abc123" --targetGuid="def456" --models="BlogPost,BlogCategory"
 
@@ -167,6 +187,15 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=pages
 
 ## Advanced Topics
 
+### List Order
+
+A sync puts each synced content list on the target in the source's item order. It runs after content (and `--syncDeletes`), and only changes a list whose order differs from the source's, including when a reorder was the only change in the source.
+
+- The new order applies to the target's staging and live list at once. No item gets a new version, so nothing needs publishing for the order to show.
+- Items that exist only on the target keep their relative order after the synced ones.
+- A scoped run (`--pages`, `--containers`, `--models`) only orders the lists its content is in, but sends every synced item of those lists, so their order stays whole.
+- `--preflight` lists the lists it would reorder under a `List order` phase, and `--jsonSummary` reports them as the `listOrder` phase.
+
 ### Auto-Publish
 
 The `--autoPublish` flag lets you automatically publish synced content and/or pages in the target instance immediately after a sync completes. Only items that are **published in the source instance** will be published in the target — staging-only items are skipped.

ci/README.md

@@ -10,14 +10,12 @@ they're available.
 
 ## When it runs
 
-| Trigger                                                          | Tier        | CLI under test                                                |
-| ---------------------------------------------------------------- | ----------- | ------------------------------------------------------------- |
-| A pull request from a branch in this repo                        | `quick`     | the PR's build. The result shows as a check on the PR         |
-| A `v*` tag push (each release, published to npm `beta`)          | `beta`      | the published package, installed from npm                     |
-| Nightly at 07:00 UTC                                             | `nightly`   | `main`. Also deletes any instances an earlier run left behind |
-| By hand (Actions tab → _Sync permutation test_ → _Run workflow_) | your choice | the branch you pick, or a published version                   |
+It leases two instances and runs for 10 to 15 minutes, so it runs only for a release, and by hand:
 
-Pull requests from forks and from Dependabot skip it, because they get no credentials.
+| Trigger                                                          | Tier        | CLI under test                                   |
+| ---------------------------------------------------------------- | ----------- | ------------------------------------------------ |
+| A `v*` tag push (a release; `npm-publish.yml` calls it)          | `beta`      | the tag's build. npm gets it only if this passes |
+| By hand (Actions tab → _Sync permutation test_ → _Run workflow_) | your choice | the branch you pick, or a published version      |
 
 From a terminal:
 
@@ -55,23 +53,30 @@ The `quick` tier runs a fixed set covering every area in two rounds. `beta` (4 r
 (6 rounds) pick changes at random from a seed printed in the summary. Replay one with
 `gh workflow run sync-permutation.yml -f tier=nightly -f seed=<seed>`.
 
-After every sync it checks:
+Every sync and preflight passes `--syncDeletes`, so a delete on one side reaches the other. After
+every sync it checks:
 
 - **sync:** exit 0 and no failed items in `--jsonSummary`;
-- **verify:** `agility verify` finds no differences;
-- **noop:** a `--preflight` in the same direction plans no creates, updates or conflicts;
 - **conflict:** the item edited on both sides ends at the synced side's value, or the sync reported a
   conflict;
 - **drift:** an edit made only on the receiving side is overwritten, or reported, never silently kept;
+- **resolve:** when the sync kept such edits, the same sync with `--overwrite` settles them, as an
+  editor would, before the checks below;
+- **verify:** `agility verify` finds no differences. A verify that finds some is tried once more a
+  minute later, because the Fetch API it reads lags behind saves (a model field rename most of all);
+- **noop:** a `--preflight` in the same direction plans no creates, updates, deletes or conflicts;
 - **apply:** the change itself could be made.
 
 Nothing stops a run early: every round runs, and each check is a row in the job summary. The JSON
 results are kept as an artifact for 30 days.
 
 ## Promotion to `latest`
 
-`npm-publish.yml` refuses to promote a version to `latest` unless that version's tag run of this
-test passed. If a tag run failed for a reason unrelated to the CLI, re-run it, then promote again.
+A `v*` tag push runs this test on the tag's build first, and `npm-publish.yml` publishes to npm
+(under `beta`) only if it passed, so every version on npm from then on has passed it. If it failed
+for a reason unrelated to the CLI, re-run the failed jobs of that tag's _Publish to npm_ run: the
+test runs again on fresh instances, then the publish. Promote a published version to `latest` with
+_Publish to npm_ → _Run workflow_.
 
 ## Cleanup
 
@@ -80,7 +85,7 @@ Instances are deleted in three ways, so a run can't leave them behind:
 1. The `release` job runs after every run, including failed and cancelled ones, and deletes every
    instance labelled with that run.
 2. Every instance expires shortly after the run's time budget and is then deleted automatically.
-3. The nightly run deletes any instance from an earlier run that is more than 6 hours old.
+3. Every run first deletes any instance from an earlier run that is more than 6 hours old.
 
 ## Running it locally
 

ci/permutation/changes.mjs

@@ -343,7 +343,11 @@ export const CHANGES = {
     const full = await ctx.c.modelMethods.getContentModel(m.id, ctx.guid);
     const f = added
       ? full.fields.find((x) => x.name === added.name)
-      : ctx.r.pick(full.fields.filter((x) => x.type === "Text"));
+      : // Not a required field: the server drops existing values on a rename, which would leave every
+        // item missing a required value, and the CLI rightly refuses to sync those
+        ctx.r.pick(
+          full.fields.filter((x) => x.type === "Text" && String(x.settings?.Required).toLowerCase() !== "true")
+        );
     if (!f) return null;
     const old = f.name;
     f.name = `${old}Renamed${tag(ctx).toUpperCase()}`;