diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 98031bd..32ef21e 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -22,19 +22,17 @@ on: default: latest jobs: - publish: - name: Build and publish to npm + # A v* tag push: check the tag, run the sync permutation test on the tag's build, and publish + # only if it passed. Nothing reaches npm (or gets a dist-tag) untested on dev. + check-version: + name: Check the tag matches package.json if: github.event_name == 'push' runs-on: ubuntu-latest permissions: contents: read - id-token: write - steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Verify tag matches package.json version run: | @@ -45,6 +43,31 @@ jobs: exit 1 fi + sync-test: + name: Sync permutation test + needs: check-version + uses: ./.github/workflows/sync-permutation.yml + with: + tier: beta + secrets: inherit + permissions: + contents: read + id-token: write + + publish: + name: Build and publish to npm + needs: sync-test + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Node.js uses: actions/setup-node@v4 with: @@ -68,8 +91,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read - # Reads the sync permutation runs for the gate below. - actions: read # Uses the trusted publisher's OIDC credentials; needs "Allow npm dist-tag" # enabled on the package's trusted publishing configuration on npmjs.com. id-token: write @@ -106,23 +127,9 @@ jobs: - name: Check the version is published run: npm view "@agility/cli@$VERSION" version | grep -qx "$VERSION" || { echo "@agility/cli@$VERSION is not on npm."; exit 1; } - # Nothing reaches `latest` untested on dev: the tag's sync permutation run - # (sync-permutation.yml, triggered by the same tag push) must have passed. If it failed - # for a reason unrelated to the CLI, re-run it from the Actions tab and promote again. - - name: Require a passing sync permutation run - if: env.DIST_TAG == 'latest' - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - SHA=$(gh api "repos/$REPO/commits/v$VERSION" --jq .sha) || { echo "No tag v$VERSION."; exit 1; } - PASSED=$(gh api "repos/$REPO/actions/workflows/sync-permutation.yml/runs?event=push&head_sha=$SHA&status=success&per_page=1" --jq .total_count) - if [ "$PASSED" = "0" ]; then - echo "No passing 'Sync permutation test' run for v$VERSION ($SHA). See the Actions tab." - exit 1 - fi - echo "v$VERSION passed the sync permutation test." - + # No separate test check: since the sync permutation test gates the publish, every version + # published from now on has passed it. (Versions published before that, 1.2.2 and earlier, + # never went through it.) - name: Promote run: npm dist-tag add "@agility/cli@$VERSION" "$DIST_TAG" diff --git a/.github/workflows/sync-permutation.yml b/.github/workflows/sync-permutation.yml index 4102164..64e6088 100644 --- a/.github/workflows/sync-permutation.yml +++ b/.github/workflows/sync-permutation.yml @@ -12,19 +12,28 @@ name: Sync permutation test # whatever happens to the rest of the run, and the leasing service deletes anything a # dead runner leaves behind once the TTL runs out. # -# Tiers (the same jobs; the tier sets TTLs and timeouts, and later slices add scenarios to it): -# quick every pull request -# beta every v* tag (the version npm-publish.yml just published under `beta`). Its result -# gates promotion to `latest` (npm-publish.yml, promote job) -# nightly every night against whatever is deployed to dev, plus a sweep for left-behind leases +# It leases two instances and runs for 10 to 15 minutes, so it runs only for a release and by hand: +# release npm-publish.yml calls it on every v* tag push, on that tag's build (tier `beta`), and +# publishes to npm only if it passes +# by hand Actions tab → Sync permutation test → Run workflow, on a branch or a published version +# +# Tiers (the same jobs; the tier sets TTLs, timeouts and the number of change rounds): +# quick two fixed rounds +# beta four seeded random rounds (a release) +# nightly six seeded random rounds on: - pull_request: - push: - tags: - - "v*" - schedule: - - cron: "0 7 * * *" # 07:00 UTC, 03:00 Eastern + workflow_call: + inputs: + tier: + type: string + default: beta + cli_version: + type: string + default: "" + seed: + type: string + default: "" workflow_dispatch: inputs: tier: @@ -42,17 +51,17 @@ on: default: "" concurrency: - group: sync-permutation-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: sync-permutation-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: false permissions: contents: read id-token: write env: - TIER: ${{ inputs.tier || (github.event_name == 'schedule' && 'nightly') || (github.ref_type == 'tag' && 'beta') || 'quick' }} - # Published version under test: the dispatch input, or the tag's version on a tag push. - CLI_VERSION: ${{ inputs.cli_version || (github.ref_type == 'tag' && github.ref_name) || '' }} + TIER: ${{ inputs.tier || 'quick' }} + # Blank: build and test this checkout (a release tests its tag's build). Otherwise a published version. + CLI_VERSION: ${{ inputs.cli_version || '' }} # Every lease of this run (and of this attempt, so a re-run gets fresh instances) starts with this. LEASE_PREFIX: cli-perm-${{ github.run_id }}-${{ github.run_attempt }}- GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -60,10 +69,6 @@ env: jobs: lease: name: Lease dev instances - # Same-repo PRs only: forks and Dependabot get no credentials. - if: >- - github.event_name != 'pull_request' || - (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') runs-on: ubuntu-latest environment: cli-dev-live timeout-minutes: 20 @@ -94,7 +99,6 @@ jobs: # Layer 4 of teardown: anything an earlier run left behind (its release job failed or never ran). - name: Sweep left-behind leases - if: env.TIER == 'nightly' run: node ci/leases.mjs sweep cli-perm- --older-than-hours 6 # TTL = the run's budget plus a margin, so the reaper is the backstop if every other layer fails. @@ -109,7 +113,7 @@ jobs: needs: lease runs-on: ubuntu-latest environment: cli-dev-live - timeout-minutes: ${{ (github.event_name == 'schedule' || inputs.tier == 'nightly') && 180 || 120 }} + timeout-minutes: ${{ inputs.tier == 'nightly' && 180 || 120 }} steps: - uses: actions/checkout@v4 @@ -133,11 +137,6 @@ jobs: if: env.CLI_VERSION != '' run: | V="${CLI_VERSION#v}" - # A tag push starts this workflow and npm-publish.yml together; wait for the publish. - for i in $(seq 1 40); do - npm view "@agility/cli@$V" version >/dev/null 2>&1 && break - echo "waiting for @agility/cli@$V on npm ($i/40)"; sleep 15 - done npm install --prefix "$RUNNER_TEMP/cli" "@agility/cli@$V" echo "CLI=node $RUNNER_TEMP/cli/node_modules/@agility/cli/dist/index.js" >> "$GITHUB_ENV" @@ -178,11 +177,7 @@ jobs: release: name: Release leases needs: [lease, test] - if: >- - always() && ( - github.event_name != 'pull_request' || - (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') - ) + if: always() runs-on: ubuntu-latest environment: cli-dev-live timeout-minutes: 10 diff --git a/README.md b/README.md index ab351a0..d13fd21 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,20 @@ agility pull [options] | `--elements` | string | `Models,Galleries,Assets,Containers,Content,Templates,Pages,Sitemaps,UrlRedirections` | Comma-separated list of elements to process | | `--models` | string | _(empty)_ | Comma-separated list of model reference names to sync (only syncs the specified models) | +#### Syncing deletes + +`--syncDeletes` adds a final phase to a sync. For every mapping whose source entity is gone from the source pull it decides: + +| Outcome | When | +| --- | --- | +| **deleted** | The source entity is deleted and the target is unchanged since the last sync (or `--overwrite` is set). The mapping is removed. | +| **conflict** (kept) | The target was edited since the last sync. Re-run with `--overwrite` to delete it anyway. | +| **blocked** (kept) | Something that stays on the target still uses it — e.g. a target-only child page, a page or item that links to the content item, items in the container, a container or item built on the model, content that embeds the asset. | +| **mapping removed** | The target is already gone, or another live source entity maps to the same target. | +| **kept** | The source item is unpublished rather than deleted, or the delete could not be confirmed. | + +Run it with `--preflight` first: the report lists every planned delete under `Delete …` phases. Deleted pages and content items go to the instance's normal delete flow (the batch is labelled "Deleted by agility-cli --syncDeletes"). + **Authentication & Environment Options:** | Option | Type | Default | Description | @@ -105,6 +119,8 @@ agility sync [options] | Option | Type | Default | Description | | --------------- | ------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--overwrite` | boolean | `false` | Conflict-scoped override. By default, when a target item has its own changes that conflict with the source, the CLI skips it to avoid data loss. With `--overwrite`, those conflicting target items are overwritten with the source version. Non-conflicting updates are applied either way. Also covers a mapped target item that no longer exists (deleted or unpublished on the target): by default that is reported as a conflict and skipped, with `--overwrite` the item is recreated and the mapping repointed. | +| `--syncDeletes` | boolean | `false` | After syncing, delete from the target the entities that were deleted in the source since the last sync (URL redirections, pages, content, containers, templates, models, assets, galleries — dependents first). Only entities the CLI previously synced (that have a mapping) are considered; nothing created directly on the target is touched. A target entity edited since the last sync is reported as a conflict and kept unless `--overwrite` is also set. An entity that something kept on the target still uses (a child page, a page or item that links to the content, a container's items, ...) is kept and reported. Content and page deletes are confirmed against the source first, so an item that was only unpublished is never deleted. Cannot be combined with `--pages`, `--containers`, `--models` or `--models-with-deps`; use `--elements` to limit the types. Works with `--preflight`. Not read from `.env`. See [Syncing deletes](#syncing-deletes). | +| `--allowMassDeletes` | boolean | `false` | With `--syncDeletes`: allow a run to delete more than half (and more than 10) of the mapped entities of one type. Without it such a run deletes nothing and fails, because a stale or partial local cache looks the same as a mass delete. | | `--autoPublish` | string | _(disabled)_ | Automatically publish synced items that were published in the source instance. Values: `content`, `pages`, `both`. If flag is provided without a value, defaults to `both`. Items that are only in staging (not published) in the source are skipped. | | `--fullPull` | boolean | `false` | Full content re-pull. Discards the stored content sync token for every locale, re-pulls all content items and pages, and removes local content/page files that no longer exist on the instance. Use when a local cache is suspected stale. Models, containers, templates, galleries and assets reconcile against the instance on every pull regardless. | | `--preflight` | boolean | `false` | Dry-run preview. Runs the full source-pull, target-pull, dependency analysis and change detection, then reports the creates/updates/skips/conflicts a real sync would produce — **without writing anything** to the target instance or mapping files. Exits non-zero when conflicts are detected, so CI can gate a real sync on a clean preflight. | @@ -141,6 +157,10 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --overwrite # Preview what a sync would do without writing anything (exits non-zero if conflicts are found) agility sync --sourceGuid="abc123" --targetGuid="def456" --preflight +# Also delete from the target what was deleted in the source since the last sync (preview first) +agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes --preflight +agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes + # Sync only specified models (models only, no dependencies) agility sync --sourceGuid="abc123" --targetGuid="def456" --models="BlogPost,BlogCategory" @@ -167,6 +187,15 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --autoPublish=pages ## Advanced Topics +### List Order + +A sync puts each synced content list on the target in the source's item order. It runs after content (and `--syncDeletes`), and only changes a list whose order differs from the source's, including when a reorder was the only change in the source. + +- The new order applies to the target's staging and live list at once. No item gets a new version, so nothing needs publishing for the order to show. +- Items that exist only on the target keep their relative order after the synced ones. +- A scoped run (`--pages`, `--containers`, `--models`) only orders the lists its content is in, but sends every synced item of those lists, so their order stays whole. +- `--preflight` lists the lists it would reorder under a `List order` phase, and `--jsonSummary` reports them as the `listOrder` phase. + ### Auto-Publish The `--autoPublish` flag lets you automatically publish synced content and/or pages in the target instance immediately after a sync completes. Only items that are **published in the source instance** will be published in the target — staging-only items are skipped. diff --git a/ci/README.md b/ci/README.md index b17c3f4..d2b3e1d 100644 --- a/ci/README.md +++ b/ci/README.md @@ -10,14 +10,12 @@ they're available. ## When it runs -| Trigger | Tier | CLI under test | -| ---------------------------------------------------------------- | ----------- | ------------------------------------------------------------- | -| A pull request from a branch in this repo | `quick` | the PR's build. The result shows as a check on the PR | -| A `v*` tag push (each release, published to npm `beta`) | `beta` | the published package, installed from npm | -| Nightly at 07:00 UTC | `nightly` | `main`. Also deletes any instances an earlier run left behind | -| By hand (Actions tab → _Sync permutation test_ → _Run workflow_) | your choice | the branch you pick, or a published version | +It leases two instances and runs for 10 to 15 minutes, so it runs only for a release, and by hand: -Pull requests from forks and from Dependabot skip it, because they get no credentials. +| Trigger | Tier | CLI under test | +| ---------------------------------------------------------------- | ----------- | ------------------------------------------------ | +| A `v*` tag push (a release; `npm-publish.yml` calls it) | `beta` | the tag's build. npm gets it only if this passes | +| By hand (Actions tab → _Sync permutation test_ → _Run workflow_) | your choice | the branch you pick, or a published version | From a terminal: @@ -55,14 +53,18 @@ The `quick` tier runs a fixed set covering every area in two rounds. `beta` (4 r (6 rounds) pick changes at random from a seed printed in the summary. Replay one with `gh workflow run sync-permutation.yml -f tier=nightly -f seed=`. -After every sync it checks: +Every sync and preflight passes `--syncDeletes`, so a delete on one side reaches the other. After +every sync it checks: - **sync:** exit 0 and no failed items in `--jsonSummary`; -- **verify:** `agility verify` finds no differences; -- **noop:** a `--preflight` in the same direction plans no creates, updates or conflicts; - **conflict:** the item edited on both sides ends at the synced side's value, or the sync reported a conflict; - **drift:** an edit made only on the receiving side is overwritten, or reported, never silently kept; +- **resolve:** when the sync kept such edits, the same sync with `--overwrite` settles them, as an + editor would, before the checks below; +- **verify:** `agility verify` finds no differences. A verify that finds some is tried once more a + minute later, because the Fetch API it reads lags behind saves (a model field rename most of all); +- **noop:** a `--preflight` in the same direction plans no creates, updates, deletes or conflicts; - **apply:** the change itself could be made. Nothing stops a run early: every round runs, and each check is a row in the job summary. The JSON @@ -70,8 +72,11 @@ results are kept as an artifact for 30 days. ## Promotion to `latest` -`npm-publish.yml` refuses to promote a version to `latest` unless that version's tag run of this -test passed. If a tag run failed for a reason unrelated to the CLI, re-run it, then promote again. +A `v*` tag push runs this test on the tag's build first, and `npm-publish.yml` publishes to npm +(under `beta`) only if it passed, so every version on npm from then on has passed it. If it failed +for a reason unrelated to the CLI, re-run the failed jobs of that tag's _Publish to npm_ run: the +test runs again on fresh instances, then the publish. Promote a published version to `latest` with +_Publish to npm_ → _Run workflow_. ## Cleanup @@ -80,7 +85,7 @@ Instances are deleted in three ways, so a run can't leave them behind: 1. The `release` job runs after every run, including failed and cancelled ones, and deletes every instance labelled with that run. 2. Every instance expires shortly after the run's time budget and is then deleted automatically. -3. The nightly run deletes any instance from an earlier run that is more than 6 hours old. +3. Every run first deletes any instance from an earlier run that is more than 6 hours old. ## Running it locally diff --git a/ci/permutation/changes.mjs b/ci/permutation/changes.mjs index b8d142c..350043a 100644 --- a/ci/permutation/changes.mjs +++ b/ci/permutation/changes.mjs @@ -343,7 +343,11 @@ export const CHANGES = { const full = await ctx.c.modelMethods.getContentModel(m.id, ctx.guid); const f = added ? full.fields.find((x) => x.name === added.name) - : ctx.r.pick(full.fields.filter((x) => x.type === "Text")); + : // Not a required field: the server drops existing values on a rename, which would leave every + // item missing a required value, and the CLI rightly refuses to sync those + ctx.r.pick( + full.fields.filter((x) => x.type === "Text" && String(x.settings?.Required).toLowerCase() !== "true") + ); if (!f) return null; const old = f.name; f.name = `${old}Renamed${tag(ctx).toUpperCase()}`; diff --git a/ci/permutation/fixtures.mjs b/ci/permutation/fixtures.mjs index c245f1a..d3d214a 100644 --- a/ci/permutation/fixtures.mjs +++ b/ci/permutation/fixtures.mjs @@ -104,6 +104,52 @@ export async function seedReferenceCycles(c, guid, locale) { }; } +// Same rule as the CLI's Auth.determineBaseUrl / determineFetchUrl: the GUID suffix picks the region. +const REGIONS = [ + ["us2", "mgmt-usa2", "api-usa2"], + ["d", "mgmt-dev", "api-dev"], + ["u", "mgmt", "api"], + ["c", "mgmt-ca", "api-ca"], + ["e", "mgmt-eu", "api-eu"], + ["a", "mgmt-aus", "api-aus"], +]; +function hosts(guid) { + const [, mgmt, api] = REGIONS.find(([suffix]) => guid.endsWith(suffix)) ?? ["", "mgmt", "api"]; + return { mgmt: `https://${mgmt}.aglty.io`, api: `https://${api}.aglty.io` }; +} + +// Content published through the Management API reaches the Fetch API's sync feed a minute or two +// later, and the CLI pulls from that feed. Wait until it lists every given item, or the timeout. +// Returns the seconds waited; throws on timeout. +export async function waitForFetchSync(token, guid, locale, ids, { timeoutMs = 10 * 60 * 1000 } = {}) { + const { mgmt, api } = hosts(guid); + const keyRes = await fetch(`${mgmt}/oauth/GetPreviewKey?guid=${guid}`, { + headers: { Authorization: `Bearer ${token}`, "Cache-Control": "no-cache" }, + }); + if (!keyRes.ok) throw new Error(`GetPreviewKey: HTTP ${keyRes.status}`); + const apiKey = (await keyRes.text()).replace(/^"|"$/g, ""); + + const started = Date.now(); + const missing = new Set(ids); + while (Date.now() - started < timeoutMs) { + // Read the whole feed from the start, a page at a time, as a full pull does + let syncToken = 0; + for (let page = 0; page < 100 && missing.size > 0; page++) { + const res = await fetch(`${api}/${guid}/preview/${locale}/sync/items?syncToken=${syncToken}&pageSize=500`, { + headers: { APIKey: apiKey, "Cache-Control": "no-cache" }, + }); + if (!res.ok) throw new Error(`Fetch API sync/items: HTTP ${res.status}`); + const body = await res.json(); + for (const item of body.items ?? []) missing.delete(item.contentID); + if (!body.items?.length || !body.syncToken || body.syncToken === syncToken) break; + syncToken = body.syncToken; + } + if (missing.size === 0) return Math.round((Date.now() - started) / 1000); + await new Promise((r) => setTimeout(r, 10_000)); + } + throw new Error(`items ${[...missing].join(", ")} not in the Fetch API sync feed after ${timeoutMs / 1000}s`); +} + // After a sync: each target copy's Next (and NextID) must hold the TARGET ID of its partner. export async function checkReferenceCycles(c, guid, locale, fixture, toTarget) { const problems = []; diff --git a/ci/permutation/run.mjs b/ci/permutation/run.mjs index bc23f28..3b38056 100644 --- a/ci/permutation/run.mjs +++ b/ci/permutation/run.mjs @@ -12,11 +12,13 @@ // // After each sync it checks: // sync exit 0, no failed items -// verify `agility verify` finds no differences -// noop a preflight in the same direction plans no creates, updates or conflicts -// conflict the both-sides item ends at the synced side's value, or the sync reported a conflict -// drift the other side's own edit was overwritten, or reported; never silently kept +// conflict the both-sides item ends at the synced side's value, or a preflight reported it +// drift the other side's own edit was overwritten, or reported as a conflict on that item +// resolve when the sync kept such edits: the same sync with --overwrite settles them +// verify `agility verify` finds no differences (verify and noop: up to 3 tries, 90s apart) +// noop a preflight in the same direction plans no creates, updates, deletes or conflicts // apply:* the change itself could be made (an API rejection is a finding too) +// Syncs and preflights pass --syncDeletes, so a delete on one side reaches the other. // Nothing stops the run early: every round runs and every result is recorded. // // Env: SOURCE_GUID, TARGET_GUID, AGILITY_TOKEN, CLI (command for the CLI under test), @@ -27,7 +29,7 @@ import { spawnSync } from "node:child_process"; import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { CHANGES, QUICK_ROUNDS, client, conflict, drift, readField, rng } from "./changes.mjs"; -import { checkReferenceCycles, seedReferenceCycles } from "./fixtures.mjs"; +import { checkReferenceCycles, seedReferenceCycles, waitForFetchSync } from "./fixtures.mjs"; const { SOURCE_GUID: A, TARGET_GUID: B, AGILITY_TOKEN, CLI } = process.env; const TIER = process.env.TIER || "quick"; @@ -80,68 +82,108 @@ function mapped(fromSourceSide) { }; } -// One sync in a direction, with a plan before it, then verify and the no-op preflight. -function syncAndCheck(round, direction) { +// Every sync and preflight also syncs deletes, so a delete on one side reaches the other. +const SYNC = ["--autoPublish=both", "--syncDeletes"]; +const PLAN = ["--preflight", "--syncDeletes"]; + +// One sync in a direction, with a plan before it and one after. The plan after shows what the sync +// left in conflict: the edits it kept on the receiving side. `step` is "sync", or "resolve" for the +// `--overwrite` sync that settles kept conflicts. +function syncStep(round, direction, step = "sync", extra = []) { const cmd = direction === "A→B" ? "sync" : "reverse-sync"; - const plan = cli(`r${round}-plan`, [cmd, ...pair, "--preflight"]); - const s = cli(`r${round}-sync`, [cmd, ...pair, "--autoPublish=both"]); + const before = step === "sync" ? cli(`r${round}-plan`, [cmd, ...pair, ...PLAN]) : null; + const s = cli(`r${round}-${step}`, [cmd, ...pair, ...SYNC, ...extra]); const failed = s.json?.totals?.failed ?? "?"; const opErr = s.json?.operationErrors?.[0]?.error; record( round, direction, - "sync", + step, s.exitCode === 0 && failed === 0, `exit ${s.exitCode}, ${s.json?.totals?.successful ?? "?"} ok, ${failed} failed${opErr ? `; ${String(opErr).slice(0, 160)}` : ""}`, s.seconds ); for (const f of (s.json?.failures ?? []).slice(0, 8)) { - record(round, direction, `sync-failure:${f.type ?? "item"}`, false, `${f.name}: ${String(f.error).slice(0, 180)}`); + record( + round, + direction, + `${step}-failure:${f.type ?? "item"}`, + false, + `${f.name}: ${String(f.error).slice(0, 180)}` + ); } + const after = step === "sync" ? cli(`r${round}-after`, [cmd, ...pair, ...PLAN]) : null; + const entries = [before, after].flatMap((x) => x?.json?.preflight?.entries ?? []); + const details = entries.filter((e) => e.action === "conflict").map((e) => e.detail ?? ""); + return { + // Whether a preflight reported a conflict on this content item (its URL is in the detail) + reported: (guid, id) => + details.some((d) => new RegExp(`/instance/${guid}/[^/]+/content/listitem-${id}(?!\\d)`).test(d)), + }; +} - const v = cli(`r${round}-verify`, ["verify", ...pair]); - const findings = v.json?.findings ?? []; - const kinds = [...new Set(findings.map((x) => x.kind ?? x.type ?? x.code).filter(Boolean))].slice(0, 6).join(", "); - record( - round, - direction, - "verify", - v.exitCode === 0, +// verify, then the no-op preflight. Both read the instances through the Fetch API, which lags behind +// the Management API: a save shows up within seconds, but an edit that creates no version (a list +// reorder, a model field rename) can take a few minutes. So when either finds something, both are +// tried again, up to three times 90s apart; a real difference survives that. +async function checkStep(round, direction) { + const cmd = direction === "A→B" ? "sync" : "reverse-sync"; + let v, n, t; + let tries = 0; + for (;;) { + tries++; + const suffix = tries === 1 ? "" : `-try${tries}`; + v = cli(`r${round}-verify${suffix}`, ["verify", ...pair]); + n = cli(`r${round}-noop${suffix}`, [cmd, ...pair, ...PLAN]); + t = n.json?.preflight?.totals; + const clean = v.exitCode === 0 && n.exitCode === 0 && !!t && !t.create && !t.update && !t.delete && !t.conflict; + if (clean || tries === 3) break; + await new Promise((res) => setTimeout(res, 90_000)); + } + const later = tries > 1 ? ` (try ${tries} of 3)` : ""; + + const findings = (v.json?.findings ?? []).filter((x) => x.severity === "failure"); + const kinds = [...new Set(findings.map((x) => x.category).filter(Boolean))].slice(0, 6).join(", "); + const outcome = { 0: "match", 1: `${findings.length || "some"} differences${kinds ? ` (${kinds})` : ""}`, 2: "could not verify" }[ v.exitCode - ] ?? `exit ${v.exitCode}`, - v.seconds - ); - - const n = cli(`r${round}-noop`, [cmd, ...pair, "--preflight"]); - const t = n.json?.preflight?.totals; + ] ?? `exit ${v.exitCode}`; + record(round, direction, "verify", v.exitCode === 0, `${outcome}${later}`, v.seconds); record( round, direction, "noop", - n.exitCode === 0 && !!t && t.create === 0 && t.update === 0 && t.conflict === 0, - t ? `would create ${t.create}, update ${t.update}, conflict ${t.conflict}` : `exit ${n.exitCode}, no plan`, + n.exitCode === 0 && !!t && t.create === 0 && t.update === 0 && t.delete === 0 && t.conflict === 0, + (t + ? `would create ${t.create}, update ${t.update}, delete ${t.delete}, conflict ${t.conflict}` + : `exit ${n.exitCode}, no plan`) + later, n.seconds ); - return { planConflicts: plan.json?.preflight?.totals?.conflict ?? 0 }; } // ── Fixtures, then round 0: first sync ──────────────────────────────────────────────────────── let cycles = null; try { cycles = await seedReferenceCycles(api, A, LOCALE); + // The first sync pulls from the Fetch API, which lists new items a minute or two after publish + const waited = await waitForFetchSync( + AGILITY_TOKEN, + A, + LOCALE, + cycles.links.map(([x]) => x) + ); record( 0, "A→B", "apply:fixture-cycles", true, - `A ↔ B and a self-reference (${cycles.links.map(([x]) => `#${x}`).join(", ")})` + `A ↔ B and a self-reference (${cycles.links.map(([x]) => `#${x}`).join(", ")}), in the Fetch API after ${waited}s` ); } catch (e) { record(0, "A→B", "apply:fixture-cycles", false, String(e?.message ?? e).slice(0, 200)); } -syncAndCheck(0, "A→B"); +syncStep(0, "A→B"); if (cycles) { const problems = await checkReferenceCycles(api, B, LOCALE, cycles, mapped(true)); @@ -154,6 +196,8 @@ if (cycles) { ); } +await checkStep(0, "A→B"); + // ── Change rounds ───────────────────────────────────────────────────────────────────────────── const names = Object.keys(CHANGES); const plan = @@ -212,7 +256,8 @@ for (let i = 0; i < plan.length; i++) { `\n── round ${round} (${direction}): ${applied.map((a) => `${a.area}/${a.change}`).join(", ") || "no changes"}` ); - const { planConflicts } = syncAndCheck(round, direction); + const { reported } = syncStep(round, direction); + let kept = false; if (both) { const after = await readField(api, to, LOCALE, both.dstId, both.field); @@ -221,13 +266,14 @@ for (let i = 0; i < plan.length; i++) { round, direction, "conflict", - synced || planConflicts > 0, + synced || reported(to, both.dstId), synced ? "the synced side's value won" - : planConflicts > 0 - ? `kept, and preflight reported ${planConflicts} conflict(s)` + : reported(to, both.dstId) + ? "kept, and reported as a conflict" : `silently kept the other side's value` ); + kept ||= !synced; } if (drifted) { const id = Number(/#(\d+)/.exec(drifted.detail)?.[1]); @@ -242,14 +288,20 @@ for (let i = 0; i < plan.length; i++) { round, direction, "drift", - same || planConflicts > 0, + same || reported(to, id), same ? "overwritten to match" - : planConflicts > 0 + : reported(to, id) ? "kept, and reported as a conflict" : "the other side's edit was silently kept; the instances differ" ); + kept ||= !same; } + + // The sync rightly kept the receiving side's own edits and reported them. Settle them the way an + // editor would, by syncing again with --overwrite, so verify and the no-op check see one state. + if (kept) syncStep(round, direction, "resolve", ["--overwrite"]); + await checkStep(round, direction); } // ── Results ─────────────────────────────────────────────────────────────────────────────────── diff --git a/package-lock.json b/package-lock.json index e42ebe1..d1f5191 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,16 @@ { "name": "@agility/cli", - "version": "1.2.2", + "version": "1.3.0-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@agility/cli", - "version": "1.2.2", + "version": "1.3.0-beta.0", "license": "ISC", "dependencies": { "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.38", + "@agility/management-sdk": "^0.2.0-beta.3", "ansi-colors": "^4.1.3", "axios": "^0.27.2", "cli-progress": "^3.11.2", @@ -89,9 +89,9 @@ } }, "node_modules/@agility/management-sdk": { - "version": "0.1.38", - "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.38.tgz", - "integrity": "sha512-g6/hNgCjf+uzcJbkPaxeWqwaAid50tMnRW0KYQ4J/fGuJqtcZG0et0X3m76Ev3yUqnkpIO/+C5zMIQMghN3/oQ==", + "version": "0.2.0-beta.3", + "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.2.0-beta.3.tgz", + "integrity": "sha512-jBe7eJxOS7HdhfEnuCBBWwPZIGuA4qR+ekwU+wKmMk8HgMVc33BhuQUSajvwSjaZFkrVhTozjYiHNQjz2Y4GVg==", "license": "MIT", "dependencies": { "axios": "^0.27.2" diff --git a/package.json b/package.json index 0aceaf4..8423344 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agility/cli", - "version": "1.2.2", + "version": "1.3.0-beta.0", "repository": { "type": "git", "url": "https://github.com/agility/agility-cli.git" @@ -53,7 +53,7 @@ }, "dependencies": { "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.38", + "@agility/management-sdk": "^0.2.0-beta.3", "ansi-colors": "^4.1.3", "axios": "^0.27.2", "cli-progress": "^3.11.2", diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 9e62f3f..c421a8c 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -88,6 +88,8 @@ export interface JsonSummaryPreflightPhase { phase: string; create: number; update: number; + /** --syncDeletes: target entities a real run would delete. */ + delete: number; skip: number; conflict: number; /** PROD-2542: would-be writes that fail validation against the target model. */ @@ -95,13 +97,13 @@ export interface JsonSummaryPreflightPhase { } export interface JsonSummaryPreflight { - totals: { create: number; update: number; skip: number; conflict: number; invalid: number }; + totals: { create: number; update: number; delete: number; skip: number; conflict: number; invalid: number }; hasConflicts: boolean; phases: JsonSummaryPreflightPhase[]; /** Every recorded action, in the order the pushers recorded them. */ entries: Array<{ phase: string; - action: "create" | "update" | "skip" | "conflict" | "invalid"; + action: "create" | "update" | "delete" | "skip" | "conflict" | "invalid"; name: string; locale?: string; detail?: string; @@ -123,6 +125,8 @@ export interface JsonSummary { options: { preflight: boolean; overwrite: boolean; + /** --syncDeletes. Optional so summaries written before the flag existed still parse. */ + syncDeletes?: boolean; autoPublish: string; elements: string[]; locales: string[]; @@ -235,6 +239,7 @@ export function emitAbortSummary( targetGuid?: string; preflight?: boolean; overwrite?: boolean; + syncDeletes?: boolean; autoPublish?: string; elements?: string; locale?: string[]; @@ -253,6 +258,7 @@ export function emitAbortSummary( options: { preflight: runState.preflight === true, overwrite: runState.overwrite === true, + syncDeletes: runState.syncDeletes === true, autoPublish: runState.autoPublish || "", elements: (runState.elements || "").split(",").filter(Boolean), locales: Array.isArray(runState.locale) ? runState.locale : [], diff --git a/src/core/push.ts b/src/core/push.ts index 038b0ce..37fc063 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -490,6 +490,7 @@ export class Push { options: { preflight: state.preflight === true, overwrite: state.overwrite === true, + syncDeletes: state.syncDeletes === true, autoPublish: state.autoPublish || "", elements: (state.elements || "").split(",").filter(Boolean), locales: Array.isArray(state.locale) ? state.locale : [], diff --git a/src/core/state.ts b/src/core/state.ts index 0305535..1500ab1 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -36,6 +36,8 @@ export interface State { // Operation control overwrite: boolean; + syncDeletes: boolean; // Delete from the target what was deleted in the source since the last sync + allowMassDeletes: boolean; // Lift the --syncDeletes mass-delete guard preflight: boolean; // Preflight mode - report planned sync actions without writing to target/mappings fullPull: boolean; // PROD-2614: discard content sync tokens and do a full content re-pull that removes stale local files jsonSummary: string; // Path to write the machine-readable run summary to; empty = disabled @@ -166,6 +168,8 @@ export const state: State = { // Operation control overwrite: false, + syncDeletes: false, + allowMassDeletes: false, preflight: false, fullPull: false, jsonSummary: "", @@ -276,6 +280,8 @@ export function setState(argv: any) { // Operation control if (argv.overwrite !== undefined) state.overwrite = argv.overwrite; + if (argv.syncDeletes !== undefined) state.syncDeletes = argv.syncDeletes; + if (argv.allowMassDeletes !== undefined) state.allowMassDeletes = argv.allowMassDeletes; // Operation control (preflight) if (argv.preflight !== undefined) state.preflight = argv.preflight; @@ -473,6 +479,8 @@ export function resetState() { // Operation control state.overwrite = false; + state.syncDeletes = false; + state.allowMassDeletes = false; state.preflight = false; state.fullPull = false; state.jsonSummary = ""; diff --git a/src/core/system-args.ts b/src/core/system-args.ts index 69aba1a..69f6cc1 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -154,6 +154,22 @@ export const systemArgs = { default: false, }, + // Sync deletes. Deliberately not primed from .env: a destructive mode is opted into per run. + syncDeletes: { + describe: + "(sync/push only) After syncing, delete from the target the entities that were deleted in the source since the last sync. Only entities the CLI previously synced (that have a mapping) are considered — nothing created directly on the target is touched. A target entity that was changed since the last sync is a conflict and is kept unless --overwrite is also set; one that something else on the target still uses is kept and reported. Content and page deletes are confirmed against the source first (an unpublished item is not deleted). Default: false.", + type: "boolean" as const, + alias: ["sync-deletes", "syncdeletes", "SyncDeletes", "SYNC_DELETES", "SYNCDELETES"], + default: false, + }, + allowMassDeletes: { + describe: + "(with --syncDeletes) Allow a sync to delete more than half of the mapped entities of a kind. Without it such a run refuses to delete anything, since a stale or partial local cache looks the same as a mass delete. Default: false.", + type: "boolean" as const, + alias: ["allow-mass-deletes", "allowmassdeletes", "AllowMassDeletes", "ALLOW_MASS_DELETES"], + default: false, + }, + // Auto-publish after sync autoPublish: { describe: @@ -267,6 +283,8 @@ export interface SystemArgs { preflight?: boolean; // Preflight mode - report planned sync actions without writing to target/mappings verbose?: boolean; overwrite?: boolean; + syncDeletes?: boolean; + allowMassDeletes?: boolean; elements?: string; guid?: string; sourceGuid?: string; diff --git a/src/core/tests/json-summary.test.ts b/src/core/tests/json-summary.test.ts index c43f391..7dd395d 100644 --- a/src/core/tests/json-summary.test.ts +++ b/src/core/tests/json-summary.test.ts @@ -181,9 +181,9 @@ describe("buildJsonSummary — shape", () => { const summary = buildJsonSummary( makeInput({ preflight: { - totals: { create: 2, update: 1, skip: 0, conflict: 1, invalid: 0 }, + totals: { create: 2, update: 1, delete: 0, skip: 0, conflict: 1, invalid: 0 }, hasConflicts: true, - phases: [{ phase: "Models", create: 2, update: 1, skip: 0, conflict: 1, invalid: 0 }], + phases: [{ phase: "Models", create: 2, update: 1, delete: 0, skip: 0, conflict: 1, invalid: 0 }], entries: [{ phase: "Models", action: "conflict", name: "LinkCard" }], }, }) @@ -395,9 +395,9 @@ describe("buildJsonSummary — required-field validation failures (PROD-2542)", it("carries the preflight invalid count additively", () => { const preflight = { - totals: { create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }, + totals: { create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }, hasConflicts: false, - phases: [{ phase: "Content", create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }], + phases: [{ phase: "Content", create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }], entries: [ { phase: "Content", action: "invalid" as const, name: "home_promobanner", locale: "en-us", detail: "x" }, { phase: "Content", action: "create" as const, name: "ok", locale: "en-us" }, diff --git a/src/core/tests/push.test.ts b/src/core/tests/push.test.ts index a4526a6..3bdc1aa 100644 --- a/src/core/tests/push.test.ts +++ b/src/core/tests/push.test.ts @@ -410,7 +410,7 @@ describe("Push.pushInstances — --jsonSummary", () => { const parsed = JSON.parse(fs.readFileSync(out, "utf8")); expect(parsed.exitCode).toBe(0); expect(process.exitCode).toBeUndefined(); - expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); expect(parsed.preflight.hasConflicts).toBe(false); expect(parsed.warnings.some((w: any) => w.code === "foreignCdnReference")).toBe(true); diff --git a/src/lib/deleters/delete-plan.ts b/src/lib/deleters/delete-plan.ts new file mode 100644 index 0000000..1c2903e --- /dev/null +++ b/src/lib/deleters/delete-plan.ts @@ -0,0 +1,334 @@ +/** + * --syncDeletes: decide what happens to each target entity whose source counterpart is gone. + * + * A candidate is a mapping record whose source entity is no longer in the source pull. Whether it + * is actually deleted on the target is decided here from facts the caller gathers (is the source + * really deleted, is the target still there, did someone edit it since the last sync, does + * anything that stays depend on it). Everything in this file is pure so the rules can be tested + * without an instance; sync-deletes.ts does the reading and the API calls. + */ + +export type DeleteKind = + | "urlRedirection" + | "page" + | "content" + | "container" + | "template" + | "model" + | "asset" + | "gallery"; + +/** Execution order: dependents before the things they depend on. */ +export const DELETE_ORDER: DeleteKind[] = [ + "urlRedirection", + "page", + "content", + "container", + "template", + "model", + "asset", + "gallery", +]; + +/** --elements name for each kind. A kind is only considered when its element is in the run. */ +export const DELETE_KIND_ELEMENT: Record = { + urlRedirection: "UrlRedirections", + page: "Pages", + content: "Content", + container: "Containers", + template: "Templates", + model: "Models", + asset: "Assets", + gallery: "Galleries", +}; + +/** Mapping folder for each kind (content and pages are per-locale). */ +export const DELETE_KIND_MAPPING_TYPE: Record = { + urlRedirection: "urlredirections", + page: "page", + content: "item", + container: "containers", + template: "templates", + model: "models", + asset: "assets", + gallery: "galleries", +}; + +/** Source/target ID fields of each kind's mapping records. */ +export const DELETE_KIND_ID_FIELDS: Record = { + urlRedirection: { source: "sourceUrlRedirectionID", target: "targetUrlRedirectionID" }, + page: { source: "sourcePageID", target: "targetPageID" }, + content: { source: "sourceContentID", target: "targetContentID" }, + container: { source: "sourceContentViewID", target: "targetContentViewID" }, + template: { source: "sourcePageTemplateID", target: "targetPageTemplateID" }, + model: { source: "sourceID", target: "targetID" }, + asset: { source: "sourceMediaID", target: "targetMediaID" }, + gallery: { source: "sourceMediaGroupingID", target: "targetMediaGroupingID" }, +}; + +export type DeleteAction = + /** Delete the target entity and drop its mapping. */ + | "delete" + /** The target was edited since the last sync; needs --overwrite. */ + | "conflict" + /** Something that is not being deleted still depends on the target entity. */ + | "blocked" + /** Nothing to delete (target already gone, or still owned by a live source); drop the mapping. */ + | "unmap" + /** The source is not confirmed deleted (e.g. only unpublished); leave everything as it is. */ + | "keep"; + +export interface DeleteCandidate { + kind: DeleteKind; + locale?: string; + sourceID: number; + targetID: number; + name: string; + /** The mapping record this candidate came from, removed again on delete/unmap. */ + record: any; + action: DeleteAction; + reason: string; + /** Pages: depth in the target sitemap, so children are deleted before their parents. */ + depth?: number; +} + +/** What the caller found out about one candidate. */ +export interface CandidateFacts { + /** true = confirmed deleted; a string = not confirmed, with the reason. */ + sourceDeleted: true | string; + targetExists: boolean; + targetChanged: boolean; + /** A live source entity that maps onto the same target. */ + sharedWithSourceID?: number; + /** Why the target cannot be deleted (what still depends on it). */ + blockedBy?: string; +} + +export function decideAction(facts: CandidateFacts, overwrite: boolean): { action: DeleteAction; reason: string } { + if (facts.sourceDeleted !== true) { + return { action: "keep", reason: facts.sourceDeleted }; + } + if (facts.sharedWithSourceID !== undefined) { + return { + action: "unmap", + reason: `target is also mapped from live source ID ${facts.sharedWithSourceID}; removing the stale mapping only`, + }; + } + if (!facts.targetExists) { + return { action: "unmap", reason: "already gone on the target; removing the mapping" }; + } + if (facts.blockedBy) { + return { action: "blocked", reason: facts.blockedBy }; + } + if (facts.targetChanged && !overwrite) { + return { + action: "conflict", + reason: "deleted in the source, but the target was changed since the last sync; use --overwrite to delete it", + }; + } + return { + action: "delete", + reason: facts.targetChanged + ? "deleted in the source (target changes discarded by --overwrite)" + : "deleted in the source", + }; +} + +export interface MappingPair { + sourceID: number; + targetID: number; + record: any; +} + +/** Usable (sourceID, targetID) pairs from a kind's mapping records. */ +export function mappingPairs(kind: DeleteKind, records: any[]): MappingPair[] { + const fields = DELETE_KIND_ID_FIELDS[kind]; + const out: MappingPair[] = []; + (records || []).forEach((record) => { + const sourceID = toId(record?.[fields.source]); + const targetID = toId(record?.[fields.target]); + if (sourceID === undefined || targetID === undefined) return; + out.push({ sourceID, targetID, record }); + }); + return out; +} + +/** + * Split a kind's mappings into the records whose source is gone (candidates) and the set of + * target IDs still owned by a live source. A target owned by a live source is never deleted. + */ +export function findCandidates( + pairs: MappingPair[], + liveSourceIDs: Set +): { candidates: MappingPair[]; liveTargets: Map } { + const candidates: MappingPair[] = []; + const liveTargets = new Map(); + pairs.forEach((pair) => { + if (liveSourceIDs.has(pair.sourceID)) { + if (!liveTargets.has(pair.targetID)) liveTargets.set(pair.targetID, pair.sourceID); + } else { + candidates.push(pair); + } + }); + return { candidates, liveTargets }; +} + +/** + * Mass-delete guard. A cache that was wiped, copied from another instance or only partly pulled + * looks exactly like a source that deleted everything. Refuse when a single kind (per locale) + * would lose more than `maxRatio` of its mapped entities and more than `minCount` of them. + */ +export const MASS_DELETE_MIN_COUNT = 10; +export const MASS_DELETE_MAX_RATIO = 0.5; + +export interface MassDeleteViolation { + kind: DeleteKind; + locale?: string; + deletes: number; + mapped: number; +} + +export function findMassDeletes( + groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }>, + minCount = MASS_DELETE_MIN_COUNT, + maxRatio = MASS_DELETE_MAX_RATIO +): MassDeleteViolation[] { + const out: MassDeleteViolation[] = []; + groups.forEach(({ kind, locale, mapped, candidates }) => { + const deletes = candidates.filter((c) => c.action === "delete").length; + if (deletes > minCount && mapped > 0 && deletes / mapped > maxRatio) { + out.push({ kind, locale, deletes, mapped }); + } + }); + return out; +} + +// ─── reference scanning ───────────────────────────────────────────────────── + +export interface FieldReferences { + contentIDs: Set; + /** Lower-cased container reference names (linked lists). */ + containers: Set; + galleryIDs: Set; +} + +/** + * Every content item, container and gallery a content item's fields point at, in any of the + * shapes the sync reads ({contentid}, {sortids}, {referencename}, {galleryid}, and the scalar + * "123" / "1,2,3" form a dropdown or checkbox list value field stores). Over-reporting only ever + * blocks a delete, so the scalar form is matched loosely. + */ +export function collectFieldReferences(fields: any): FieldReferences { + const refs: FieldReferences = { contentIDs: new Set(), containers: new Set(), galleryIDs: new Set() }; + const visit = (value: any) => { + if (value === null || value === undefined) return; + if (typeof value === "string") { + if (/^\s*\d+(\s*,\s*\d+)*\s*$/.test(value)) parseIdList(value).forEach((id) => refs.contentIDs.add(id)); + return; + } + if (Array.isArray(value)) { + value.forEach(visit); + return; + } + if (typeof value !== "object") return; + Object.keys(value).forEach((key) => { + const lowered = key.toLowerCase(); + const v = value[key]; + if (lowered === "contentid" || lowered === "sortids") { + parseIdList(v).forEach((id) => refs.contentIDs.add(id)); + } else if (lowered === "referencename" && typeof v === "string" && v) { + refs.containers.add(v.toLowerCase()); + } else if (lowered === "galleryid") { + parseIdList(v).forEach((id) => refs.galleryIDs.add(id)); + } else { + visit(v); + } + }); + }; + visit(fields); + return refs; +} + +export function parseIdList(value: any): number[] { + if (typeof value === "number") return value > 0 ? [value] : []; + if (typeof value !== "string") return []; + return value + .split(",") + .map((s) => parseInt(s.trim(), 10)) + .filter((n) => Number.isFinite(n) && n > 0); +} + +export function toId(value: any): number | undefined { + const n = typeof value === "string" ? parseInt(value, 10) : value; + return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : undefined; +} + +/** Content IDs placed as modules in a page's zones. */ +export function pageZoneContentIDs(page: any): number[] { + const out: number[] = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (!Array.isArray(modules)) return; + modules.forEach((module) => { + const item = module?.item; + const id = toId(item?.contentid ?? item?.contentID ?? item?.ContentID); + if (id) out.push(id); + }); + }); + return out; +} + +/** Lower-cased model reference names of the modules placed in a page's zones. */ +export function pageZoneModuleNames(page: any): string[] { + const out: string[] = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (!Array.isArray(modules)) return; + modules.forEach((module) => { + if (typeof module?.module === "string" && module.module) out.push(module.module.toLowerCase()); + }); + }); + return out; +} + +/** Lower-cased model reference names a model's fields link to (Content / linked-list fields). */ +export function modelFieldModelReferences(model: any): string[] { + const out: string[] = []; + (model?.fields || []).forEach((field: any) => { + const settings = field?.settings || {}; + Object.keys(settings).forEach((key) => { + if (key.toLowerCase() !== "contentdefinition") return; + const v = settings[key]; + if (typeof v === "string" && v) out.push(v.toLowerCase()); + }); + }); + return out; +} + +/** "a, b and 3 more" for blocker messages. */ +export function describeList(names: string[], max = 3): string { + const unique = Array.from(new Set(names)); + if (unique.length <= max) return unique.join(", "); + return `${unique.slice(0, max).join(", ")} and ${unique.length - max} more`; +} + +/** Flatten a sitemap's page tree into parent links and depths. */ +export function indexSitemap(channels: any[]): { parentOf: Map; depthOf: Map } { + const parentOf = new Map(); + const depthOf = new Map(); + const walk = (nodes: any[], parent: number | null, depth: number) => { + (nodes || []).forEach((node) => { + const id = toId(node?.pageID); + if (id === undefined) return; + parentOf.set(id, parent); + depthOf.set(id, depth); + walk(node?.childPages || [], id, depth + 1); + }); + }; + (channels || []).forEach((channel) => walk(channel?.pages || [], null, 0)); + return { parentOf, depthOf }; +} diff --git a/src/lib/deleters/sync-deletes.ts b/src/lib/deleters/sync-deletes.ts new file mode 100644 index 0000000..eb40815 --- /dev/null +++ b/src/lib/deleters/sync-deletes.ts @@ -0,0 +1,992 @@ +/** + * --syncDeletes: after every pusher has run, delete from the target what was deleted in the + * source since the last sync. + * + * Only entities reached through a mapping record are ever considered — something created on the + * target is never touched. For each mapping whose source entity is gone from the source pull: + * + * 1. confirm the source really is deleted. Content and pages are checked against the source + * Management API, because a full pull also leaves out unpublished items; the other kinds are + * pulled as complete lists every run, so absence (or isDeleted) is authoritative — except an + * entirely empty list, which is treated as a bad pull rather than "everything deleted"; + * 2. read the target's current state (content and pages from the API, since this run's pushes + * happened after the pull) and whether it was edited since the last sync — a conflict unless + * --overwrite; + * 3. block the delete when anything that stays on the target still depends on it (a page's + * children, content placed on a page or linked from other content, a container's items, ...); + * 4. refuse the whole delete phase when one kind would lose more than half its mapped entities + * (a wiped or stale cache looks exactly like that), unless --allowMassDeletes. + * + * Deletes run dependents-first (DELETE_ORDER). Each delete — and each mapping whose target is + * already gone — removes its mapping record. Under --preflight the plan is reported and nothing + * is written. + */ +import ansiColors from "ansi-colors"; +import { parse } from "date-fns"; +import { state, getApiClient, getLoggerForGuid } from "../../core/state"; +import { fileOperations } from "../../core/fileOperations"; +import { GuidDataLoader, GuidEntities } from "../pushers/guid-data-loader"; +import { preflightReport } from "../preflight/preflight-report"; +import { MAX_URL_REDIRECTIONS_PER_REQUEST } from "@agility/management-sdk"; +import { FailureDetail, FailureDetailType, PusherResult } from "../../types/sourceData"; +import { + CandidateFacts, + DeleteCandidate, + DeleteKind, + DELETE_KIND_ELEMENT, + DELETE_KIND_ID_FIELDS, + DELETE_KIND_MAPPING_TYPE, + DELETE_ORDER, + collectFieldReferences, + decideAction, + describeList, + findCandidates, + findMassDeletes, + indexSitemap, + mappingPairs, + MappingPair, + MASS_DELETE_MAX_RATIO, + MASS_DELETE_MIN_COUNT, + modelFieldModelReferences, + pageZoneContentIDs, + pageZoneModuleNames, + toId, +} from "./delete-plan"; + +const DELETED_STATE = 3; +const UNPUBLISHED_STATE = 7; +const API_CONCURRENCY = 6; +const SYSTEM_CONTAINER_NAMES = new Set([ + "agilitycssfiles", + "agilityjavascriptfiles", + "agilityglobalcodetemplates", + "agilitymodulecodetemplates", + "agilitypagecodetemplates", +]); + +const PHASE_LABEL: Record = { + urlRedirection: "Delete URL Redirections", + page: "Delete Pages", + content: "Delete Content", + container: "Delete Containers", + template: "Delete Templates", + model: "Delete Models", + asset: "Delete Assets", + gallery: "Delete Galleries", +}; + +const FAILURE_TYPE: Record = { + urlRedirection: "urlRedirection", + page: "page", + content: "content", + container: "container", + template: "template", + model: "model", + asset: "asset", + gallery: "gallery", +}; + +export interface SyncDeletesOptions { + sourceGuid: string; + targetGuid: string; + locales: string[]; + elements: string[]; +} + +export interface SyncDeletesResult extends PusherResult { + candidates: DeleteCandidate[]; + massDeleteRefused: boolean; +} + +interface LocaleSide { + content: any[]; + pages: any[]; +} + +interface Side { + guid: string; + entities: GuidEntities; + byLocale: Map; + urlRedirectionsPulled: boolean; +} + +/** Everything one planning pass reads, shared by the per-kind planners. */ +interface PlanContext { + opts: SyncDeletesOptions; + source: Side; + target: Side; + overwrite: boolean; + /** Mapping records per kind (and per locale for content/pages). */ + mappings: Map; + /** Target IDs mapped from a live source, per kind/locale — push keeps these in line with the source. */ + liveTargets: Map>; + /** Target IDs this run will delete, per kind/locale (filled in as kinds are planned). */ + deleting: Map>; + /** Live (post-push) target sitemap per locale; null when it could not be read. */ + targetSitemaps: Map; depthOf: Map } | null>; +} + +function groupKey(kind: DeleteKind, locale?: string): string { + return locale ? `${kind}:${locale}` : kind; +} + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +function errorStatus(error: any): number | undefined { + return error?.innerError?.response?.status ?? error?.response?.status ?? error?.status; +} + +function errorMessage(error: any): string { + const data = error?.innerError?.response?.data; + if (typeof data === "string" && data) return data; + if (data?.message) return data.message; + return error?.message || String(error); +} + +async function mapPool(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await fn(items[index]); + } + }); + await Promise.all(workers); + return results; +} + +// ─── loading ───────────────────────────────────────────────────────────────── + +async function loadSide(guid: string, locales: string[]): Promise { + const loader = new GuidDataLoader(guid); + const entities = await loader.loadCompleteGuidEntities(locales[0]); + const byLocale = new Map(); + for (const locale of locales) { + const localeEntities = locale === locales[0] ? entities : await loader.loadCompleteGuidEntities(locale); + byLocale.set(locale, { content: localeEntities.content || [], pages: localeEntities.pages || [] }); + } + const redirectionsFile = new fileOperations(guid, locales[0]).readJsonFile("urlredirections/urlredirections.json"); + return { guid, entities, byLocale, urlRedirectionsPulled: !!redirectionsFile }; +} + +function readMappings(kind: DeleteKind, opts: SyncDeletesOptions, locale?: string): any[] { + const fileOps = new fileOperations(opts.targetGuid); + const records = fileOps.getMappingFile(DELETE_KIND_MAPPING_TYPE[kind], opts.sourceGuid, opts.targetGuid, locale ?? null); + return Array.isArray(records) ? records : []; +} + +// ─── API reads ─────────────────────────────────────────────────────────────── + +/** Is a content item / page really deleted in the source, or only missing from the pull? */ +async function confirmSourceDeleted(kind: "content" | "page", sourceID: number, locale: string): Promise { + const api = getApiClient(); + try { + const entity = + kind === "content" + ? await api.contentMethods.getContentItem(sourceID, state.sourceGuid, locale) + : await api.pageMethods.getPage(sourceID, state.sourceGuid, locale); + const itemState = entity?.properties?.state; + if (!entity || itemState === DELETED_STATE) return true; + if (itemState === UNPUBLISHED_STATE) return "unpublished in the source, not deleted"; + return `still exists in the source (state ${itemState}) but is missing from the pull; re-run with --fullPull`; + } catch (error: any) { + if (errorStatus(error) === 404) return true; + return `could not confirm the source delete: ${errorMessage(error)}`; + } +} + +/** The target content item / page as it is now (after this run's pushes). */ +async function readTargetNow( + kind: "content" | "page", + targetID: number, + locale: string +): Promise<{ exists: boolean; entity?: any; error?: string }> { + const api = getApiClient(); + try { + const entity = + kind === "content" + ? await api.contentMethods.getContentItem(targetID, state.targetGuid, locale) + : await api.pageMethods.getPage(targetID, state.targetGuid, locale); + if (!entity || entity?.properties?.state === DELETED_STATE) return { exists: false }; + return { exists: true, entity }; + } catch (error: any) { + if (errorStatus(error) === 404) return { exists: false }; + return { exists: true, error: errorMessage(error) }; + } +} + +// ─── planning ──────────────────────────────────────────────────────────────── + +function newCandidate(kind: DeleteKind, pair: MappingPair, name: string, locale?: string): DeleteCandidate { + return { kind, locale, sourceID: pair.sourceID, targetID: pair.targetID, name, record: pair.record, action: "keep", reason: "" }; +} + +function apply(candidate: DeleteCandidate, facts: CandidateFacts, overwrite: boolean): DeleteCandidate { + const { action, reason } = decideAction(facts, overwrite); + candidate.action = action; + candidate.reason = reason; + return candidate; +} + +/** Is this target entity going away in this run, or kept in line with a live source by the push? */ +function isDeleting(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return ctx.deleting.get(groupKey(kind, locale))?.has(targetID) === true; +} + +function isLiveMapped(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return ctx.liveTargets.get(groupKey(kind, locale))?.has(targetID) === true; +} + +/** A target entity that stays after this run and is not maintained from a live source. */ +function stays(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return !isDeleting(ctx, kind, targetID, locale) && !isLiveMapped(ctx, kind, targetID, locale); +} + +function recordDeleting(ctx: PlanContext, kind: DeleteKind, candidates: DeleteCandidate[], locale?: string) { + const key = groupKey(kind, locale); + const set = ctx.deleting.get(key) ?? new Set(); + candidates.filter((c) => c.action === "delete").forEach((c) => set.add(c.targetID)); + ctx.deleting.set(key, set); +} + +/** + * Re-check blockers until nothing changes: blocking one delete can keep alive something another + * candidate depends on (a page whose child is in conflict, content linked from a kept item). + */ +function settleBlocking( + ctx: PlanContext, + kind: DeleteKind, + candidates: DeleteCandidate[], + blockerOf: (candidate: DeleteCandidate) => string | undefined, + locale?: string +): void { + let changed = true; + while (changed) { + changed = false; + ctx.deleting.set(groupKey(kind, locale), new Set(candidates.filter((c) => c.action === "delete").map((c) => c.targetID))); + for (const candidate of candidates) { + if (candidate.action !== "delete" && candidate.action !== "conflict") continue; + const blocker = blockerOf(candidate); + if (blocker) { + candidate.action = "blocked"; + candidate.reason = blocker; + changed = true; + } + } + } + recordDeleting(ctx, kind, candidates, locale); +} + +function prepareKind( + ctx: PlanContext, + kind: DeleteKind, + liveSourceIDs: Set, + locale?: string +): { pairs: MappingPair[]; candidates: MappingPair[] } { + const records = readMappings(kind, ctx.opts, locale); + ctx.mappings.set(groupKey(kind, locale), records); + const pairs = mappingPairs(kind, records); + const { candidates, liveTargets } = findCandidates(pairs, liveSourceIDs); + ctx.liveTargets.set(groupKey(kind, locale), liveTargets); + return { pairs, candidates }; +} + +function sharedWith(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): number | undefined { + return ctx.liveTargets.get(groupKey(kind, locale))?.get(targetID); +} + +async function planUrlRedirections(ctx: PlanContext): Promise { + const live = new Set((ctx.source.entities.urlRedirections || []).map((r: any) => r?.id).filter(Boolean)); + const { candidates } = prepareKind(ctx, "urlRedirection", live); + const targetIds = new Set((ctx.target.entities.urlRedirections || []).map((r: any) => r?.id)); + const emptySource = !ctx.source.urlRedirectionsPulled || live.size === 0; + return candidates.map((pair) => { + const candidate = newCandidate("urlRedirection", pair, pair.record?.originUrl || `redirection ${pair.targetID}`); + return apply( + candidate, + { + sourceDeleted: emptySource ? "the source URL redirection list is empty or was not pulled; not treating that as deletes" : true, + // An unpulled target list can't prove absence; the delete endpoint reports notFound instead. + targetExists: !ctx.target.urlRedirectionsPulled || targetIds.has(pair.targetID), + targetChanged: false, + sharedWithSourceID: sharedWith(ctx, "urlRedirection", pair.targetID), + }, + ctx.overwrite + ); + }); +} + +async function planPages(ctx: PlanContext, locale: string): Promise { + const sourcePages = ctx.source.byLocale.get(locale)?.pages || []; + const live = new Set(sourcePages.map((p: any) => toId(p?.pageID)).filter(Boolean) as number[]); + const { candidates } = prepareKind(ctx, "page", live, locale); + if (candidates.length === 0) return []; + const targetPagesById = new Map(); + (ctx.target.byLocale.get(locale)?.pages || []).forEach((p: any) => targetPagesById.set(p?.pageID, p)); + + const planned = await mapPool(candidates, API_CONCURRENCY, async (pair) => { + const cached = targetPagesById.get(pair.targetID); + const shared = sharedWith(ctx, "page", pair.targetID, locale); + const sourceDeleted = + live.size === 0 + ? "the source page list for this locale is empty; not treating that as deletes" + : shared !== undefined + ? true + : await confirmSourceDeleted("page", pair.sourceID, locale); + const target = sourceDeleted === true && shared === undefined ? await readTargetNow("page", pair.targetID, locale) : { exists: true }; + const entity = (target as any).entity ?? cached; + const name = entity?.name || entity?.title || `page ${pair.targetID}`; + const candidate = newCandidate("page", pair, name, locale); + const targetVersion = entity?.properties?.versionID ?? 0; + return apply( + candidate, + { + sourceDeleted, + targetExists: target.exists, + targetChanged: targetVersion > (pair.record?.targetVersionID ?? 0), + sharedWithSourceID: shared, + blockedBy: (target as any).error ? `could not read the target page: ${(target as any).error}` : undefined, + }, + ctx.overwrite + ); + }); + + const sitemap = ctx.targetSitemaps.get(locale); + const childrenOf = new Map(); + sitemap?.parentOf.forEach((parent, id) => { + if (parent === null) return; + const list = childrenOf.get(parent) ?? []; + list.push(id); + childrenOf.set(parent, list); + }); + + settleBlocking( + ctx, + "page", + planned, + (candidate) => { + if (!sitemap) return "could not read the target sitemap to check for child pages"; + // The live sitemap reflects this run's page moves, so every child that is not itself being + // deleted keeps the parent — mapped or not. + const remaining = (childrenOf.get(candidate.targetID) || []).filter((id) => !isDeleting(ctx, "page", id, locale)); + if (remaining.length === 0) return undefined; + const names = remaining.map((id) => targetPagesById.get(id)?.name || `page ${id}`); + return `still has child pages on the target: ${describeList(names)}`; + }, + locale + ); + planned.forEach((c) => (c.depth = sitemap?.depthOf.get(c.targetID) ?? 0)); + return planned; +} + +async function planContent(ctx: PlanContext, locale: string): Promise { + const sourceContent = ctx.source.byLocale.get(locale)?.content || []; + const live = new Set(sourceContent.map((c: any) => toId(c?.contentID)).filter(Boolean) as number[]); + const { candidates } = prepareKind(ctx, "content", live, locale); + if (candidates.length === 0) return []; + + const targetContent = ctx.target.byLocale.get(locale)?.content || []; + const targetPages = ctx.target.byLocale.get(locale)?.pages || []; + const contentById = new Map(); + targetContent.forEach((c: any) => contentById.set(c?.contentID, c)); + + // Who points at each content item on the target: other items' fields, and page zones. + const referencedBy = new Map>(); + const addRef = (contentID: number, ref: { kind: "content" | "page"; id: number; name: string }) => { + const list = referencedBy.get(contentID) ?? []; + list.push(ref); + referencedBy.set(contentID, list); + }; + targetContent.forEach((item: any) => { + const name = `${item?.properties?.referenceName || "item"} #${item?.contentID}`; + collectFieldReferences(item?.fields).contentIDs.forEach((id) => { + if (id !== item?.contentID) addRef(id, { kind: "content", id: item.contentID, name }); + }); + }); + targetPages.forEach((page: any) => { + pageZoneContentIDs(page).forEach((id) => addRef(id, { kind: "page", id: page.pageID, name: page?.name || `page ${page.pageID}` })); + }); + + const planned = await mapPool(candidates, API_CONCURRENCY, async (pair) => { + const cached = contentById.get(pair.targetID); + const shared = sharedWith(ctx, "content", pair.targetID, locale); + const sourceDeleted = + live.size === 0 + ? "the source content list for this locale is empty; not treating that as deletes" + : shared !== undefined + ? true + : await confirmSourceDeleted("content", pair.sourceID, locale); + const target = + sourceDeleted === true && shared === undefined ? await readTargetNow("content", pair.targetID, locale) : { exists: true }; + const entity = (target as any).entity ?? cached; + const name = `${entity?.properties?.referenceName || "content"} #${pair.targetID}`; + const candidate = newCandidate("content", pair, name, locale); + const targetVersion = entity?.properties?.versionID ?? 0; + return apply( + candidate, + { + sourceDeleted, + targetExists: target.exists, + targetChanged: targetVersion > (pair.record?.targetVersionID ?? 0), + sharedWithSourceID: shared, + blockedBy: (target as any).error ? `could not read the target item: ${(target as any).error}` : undefined, + }, + ctx.overwrite + ); + }); + + settleBlocking( + ctx, + "content", + planned, + (candidate) => { + const holders = (referencedBy.get(candidate.targetID) || []).filter((ref) => stays(ctx, ref.kind, ref.id, locale)); + if (holders.length === 0) return undefined; + return `still used on the target by ${describeList(holders.map((h) => `${h.kind} ${h.name}`))}`; + }, + locale + ); + return planned; +} + +/** Container (and other guid-level) liveness: in the pulled list and not flagged deleted. */ +function liveIds(list: any[], idOf: (e: any) => any): Set { + const out = new Set(); + (list || []).forEach((e) => { + if (e?.isDeleted === true) return; + const id = toId(idOf(e)); + if (id) out.add(id); + }); + return out; +} + +function emptyListReason(label: string): string { + return `the source ${label} list is empty; not treating that as deletes`; +} + +function changedSince(current: any, recorded: any, format?: string): boolean { + if (!current || !recorded) return false; + const a = format ? parse(current, format, new Date()) : new Date(current); + const b = format ? parse(recorded, format, new Date()) : new Date(recorded); + if (isNaN(a.getTime()) || isNaN(b.getTime())) return false; + return a > b; +} + +function planContainers(ctx: PlanContext): DeleteCandidate[] { + const sourceList = ctx.source.entities.containers || []; + const live = liveIds(sourceList, (c) => c?.contentViewID); + const { candidates } = prepareKind(ctx, "container", live); + const targetById = new Map(); + (ctx.target.entities.containers || []).forEach((c: any) => { + if (c?.isDeleted !== true) targetById.set(c?.contentViewID, c); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const refName = lower(target?.referenceName || pair.record?.targetReferenceName); + const candidate = newCandidate("container", pair, target?.referenceName || pair.record?.targetReferenceName || `container ${pair.targetID}`); + + let blockedBy: string | undefined; + if (SYSTEM_CONTAINER_NAMES.has(refName)) blockedBy = "system container"; + if (!blockedBy && target) { + const holders: string[] = []; + ctx.opts.locales.forEach((locale) => { + const side = ctx.target.byLocale.get(locale); + (side?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + const itemName = `content #${item.contentID} [${locale}]`; + if (lower(item?.properties?.referenceName) === refName) holders.push(itemName); + else if (collectFieldReferences(item?.fields).containers.has(refName)) holders.push(`${itemName} (linked list)`); + }); + (side?.pages || []).forEach((page: any) => { + if (lower(page?.dynamic?.referenceName) === refName && stays(ctx, "page", page.pageID, locale)) { + holders.push(`dynamic page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + if (holders.length > 0) blockedBy = `still used on the target by ${describeList(holders)}`; + } + + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("container") : true, + targetExists: !!target, + targetChanged: changedSince(target?.lastModifiedDate, pair.record?.targetLastModifiedDate, "MM/dd/yyyy hh:mma"), + sharedWithSourceID: sharedWith(ctx, "container", pair.targetID), + blockedBy, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "container", planned); + return planned; +} + +function planTemplates(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.templates || [], (t) => t?.pageTemplateID); + const { candidates } = prepareKind(ctx, "template", live); + const targetById = new Map(); + (ctx.target.entities.templates || []).forEach((t: any) => { + if (t?.isDeleted !== true) targetById.set(t?.pageTemplateID, t); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const templateName = target?.pageTemplateName || pair.record?.targetPageTemplateName; + const candidate = newCandidate("template", pair, templateName || `template ${pair.targetID}`); + const holders: string[] = []; + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.pages || []).forEach((page: any) => { + if (lower(page?.templateName) === lower(templateName) && stays(ctx, "page", page.pageID, locale)) { + holders.push(`page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("page template") : true, + targetExists: !!target, + targetChanged: false, + sharedWithSourceID: sharedWith(ctx, "template", pair.targetID), + blockedBy: holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "template", planned); + return planned; +} + +function planModels(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.models || [], (m) => m?.id); + const { candidates } = prepareKind(ctx, "model", live); + const targetModels = ctx.target.entities.models || []; + const targetById = new Map(); + targetModels.forEach((m: any) => targetById.set(m?.id, m)); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const refName = target?.referenceName || pair.record?.targetReferenceName; + const candidate = newCandidate("model", pair, refName || `model ${pair.targetID}`); + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("model") : true, + targetExists: !!target, + targetChanged: changedSince(target?.lastModifiedDate, pair.record?.targetLastModifiedDate), + sharedWithSourceID: sharedWith(ctx, "model", pair.targetID), + }, + ctx.overwrite + ); + }); + + settleBlocking(ctx, "model", planned, (candidate) => { + const target = targetById.get(candidate.targetID); + const refName = lower(target?.referenceName || candidate.record?.targetReferenceName); + const holders: string[] = []; + (ctx.target.entities.containers || []).forEach((c: any) => { + if (c?.isDeleted === true || c?.contentDefinitionID !== candidate.targetID) return; + if (stays(ctx, "container", c.contentViewID)) holders.push(`container ${c.referenceName}`); + }); + ctx.opts.locales.forEach((locale) => { + const side = ctx.target.byLocale.get(locale); + (side?.content || []).forEach((item: any) => { + if (lower(item?.properties?.definitionName) === refName && stays(ctx, "content", item.contentID, locale)) { + holders.push(`content #${item.contentID} [${locale}]`); + } + }); + (side?.pages || []).forEach((page: any) => { + if (pageZoneModuleNames(page).includes(refName) && stays(ctx, "page", page.pageID, locale)) { + holders.push(`page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + targetModels.forEach((m: any) => { + if (m?.id === candidate.targetID || !stays(ctx, "model", m?.id)) return; + if (modelFieldModelReferences(m).includes(refName)) holders.push(`model ${m.referenceName}`); + }); + return holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined; + }); + return planned; +} + +function planAssets(ctx: PlanContext): DeleteCandidate[] { + const isFile = (a: any) => a?.isFolder !== true; + const live = liveIds((ctx.source.entities.assets || []).filter(isFile), (a) => a?.mediaID); + const { candidates } = prepareKind(ctx, "asset", live); + const targetById = new Map(); + (ctx.target.entities.assets || []).forEach((a: any) => { + if (a?.isDeleted !== true && isFile(a)) targetById.set(a?.mediaID, a); + }); + + // Content that stays, as one searchable string per item, built once. + const remainingContent: Array<{ name: string; text: string }> = []; + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + remainingContent.push({ name: `content #${item.contentID} [${locale}]`, text: JSON.stringify(item?.fields ?? {}) }); + }); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const candidate = newCandidate("asset", pair, target?.fileName || pair.record?.targetUrl || `asset ${pair.targetID}`); + let blockedBy: string | undefined; + if (target) { + const urls = [target.url, target.originUrl, target.edgeUrl, pair.record?.targetUrl].filter( + (u): u is string => typeof u === "string" && u.length > 0 + ); + const holders = remainingContent.filter((c) => urls.some((u) => c.text.includes(u))).map((c) => c.name); + if (holders.length > 0) blockedBy = `still used on the target by ${describeList(holders)}`; + } + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("asset") : true, + targetExists: !!target, + targetChanged: changedSince(target?.dateModified, pair.record?.targetDateModified), + sharedWithSourceID: sharedWith(ctx, "asset", pair.targetID), + blockedBy, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "asset", planned); + return planned; +} + +function planGalleries(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.galleries || [], (g) => g?.mediaGroupingID); + const { candidates } = prepareKind(ctx, "gallery", live); + const targetById = new Map(); + (ctx.target.entities.galleries || []).forEach((g: any) => { + if (g?.isDeleted !== true) targetById.set(g?.mediaGroupingID, g); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const candidate = newCandidate("gallery", pair, target?.name || `gallery ${pair.targetID}`); + const holders: string[] = []; + if (target) { + (ctx.target.entities.assets || []).forEach((a: any) => { + if (a?.isDeleted === true || a?.mediaGroupingID !== pair.targetID) return; + if (stays(ctx, "asset", a.mediaID)) holders.push(`asset ${a.fileName || a.mediaID}`); + }); + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + if (collectFieldReferences(item?.fields).galleryIDs.has(pair.targetID)) { + holders.push(`content #${item.contentID} [${locale}]`); + } + }); + }); + } + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("gallery") : true, + targetExists: !!target, + targetChanged: changedSince(target?.modifiedOn, pair.record?.targetModifiedOn, "MM/dd/yyyy hh:mma"), + sharedWithSourceID: sharedWith(ctx, "gallery", pair.targetID), + blockedBy: holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "gallery", planned); + return planned; +} + +async function loadTargetSitemaps(ctx: PlanContext): Promise { + const api = getApiClient(); + for (const locale of ctx.opts.locales) { + try { + const channels = await api.pageMethods.getSitemap(ctx.opts.targetGuid, locale); + ctx.targetSitemaps.set(locale, indexSitemap(channels as any[])); + } catch { + ctx.targetSitemaps.set(locale, null); + } + } +} + +/** Build the full delete plan. Exported for tests; reads the pulled caches and the mapping files. */ +export async function planSyncDeletes(opts: SyncDeletesOptions): Promise<{ groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }> }> { + const ctx: PlanContext = { + opts, + source: await loadSide(opts.sourceGuid, opts.locales), + target: await loadSide(opts.targetGuid, opts.locales), + overwrite: state.overwrite === true, + mappings: new Map(), + liveTargets: new Map(), + deleting: new Map(), + targetSitemaps: new Map(), + }; + const wants = (kind: DeleteKind) => opts.elements.includes(DELETE_KIND_ELEMENT[kind]); + const groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }> = []; + const add = (kind: DeleteKind, candidates: DeleteCandidate[], locale?: string) => { + groups.push({ kind, locale, mapped: (ctx.mappings.get(groupKey(kind, locale)) || []).length, candidates }); + }; + + if (wants("page")) await loadTargetSitemaps(ctx); + + // Planned in execution order: each kind's blockers look at what the kinds before it delete. + if (wants("urlRedirection")) add("urlRedirection", await planUrlRedirections(ctx)); + for (const locale of opts.locales) { + if (wants("page")) add("page", await planPages(ctx, locale), locale); + } + for (const locale of opts.locales) { + if (wants("content")) add("content", await planContent(ctx, locale), locale); + } + if (wants("container")) add("container", planContainers(ctx)); + if (wants("template")) add("template", planTemplates(ctx)); + if (wants("model")) add("model", planModels(ctx)); + if (wants("asset")) add("asset", planAssets(ctx)); + if (wants("gallery")) add("gallery", planGalleries(ctx)); + + return { groups }; +} + +// ─── execution ─────────────────────────────────────────────────────────────── + +async function waitForBatch(batchID: number, guid: string): Promise { + const api = getApiClient(); + for (let attempt = 0; attempt < 150; attempt++) { + const batch: any = await api.batchMethods.getBatch(batchID, guid, true); + const batchState = batch?.batchState ?? batch?.BatchState; + if (batchState === 3) { + const errorData = batch?.errorData ?? batch?.ErrorData; + const itemError = (batch?.items ?? batch?.Items ?? []).find((i: any) => i?.errorMessage)?.errorMessage; + if (errorData || itemError) throw new Error(`batch ${batchID} failed: ${itemError || errorData}`); + return; + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + throw new Error(`batch ${batchID} did not finish in time; check the Batches page in the Content Manager`); +} + +async function deleteOne(candidate: DeleteCandidate, opts: SyncDeletesOptions): Promise { + const api = getApiClient(); + const guid = state.targetGuid; + switch (candidate.kind) { + case "page": + case "content": { + // Deleting a page can take its zone content with it, and a delete in one locale can reach + // another: re-read right before deleting so an already-gone target counts as done. + const now = await readTargetNow(candidate.kind, candidate.targetID, candidate.locale!); + if (!now.exists) return; + break; + } + } + switch (candidate.kind) { + case "page": { + const [batchID] = await api.pageMethods.deletePage(candidate.targetID, guid, candidate.locale!, "Deleted by agility-cli --syncDeletes", true); + await waitForBatch(batchID, guid); + return; + } + case "content": { + const [batchID] = await api.contentMethods.deleteContent(candidate.targetID, guid, candidate.locale!, "Deleted by agility-cli --syncDeletes", true); + await waitForBatch(batchID, guid); + return; + } + case "container": + await api.containerMethods.deleteContainer(candidate.targetID, guid); + return; + case "template": + await api.pageMethods.deletePageTemplate(guid, opts.locales[0], candidate.targetID); + return; + case "model": + await api.modelMethods.deleteModel(candidate.targetID, guid); + return; + case "asset": + await api.assetMethods.deleteFile(candidate.targetID, guid); + return; + case "gallery": + await api.assetMethods.deleteGallery(guid, candidate.targetID); + return; + default: + throw new Error(`unsupported delete kind ${candidate.kind}`); + } +} + +/** Execute one group's deletes; returns the candidates whose delete failed. */ +async function executeGroup(candidates: DeleteCandidate[], opts: SyncDeletesOptions): Promise> { + const failures = new Map(); + const toDelete = candidates.filter((c) => c.action === "delete"); + if (toDelete.length === 0) return failures; + const kind = toDelete[0].kind; + + if (kind === "urlRedirection") { + const api = getApiClient(); + for (let i = 0; i < toDelete.length; i += MAX_URL_REDIRECTIONS_PER_REQUEST) { + const chunk = toDelete.slice(i, i + MAX_URL_REDIRECTIONS_PER_REQUEST); + try { + // notFound means it is already gone, which is the outcome we wanted. + await api.urlRedirectionMethods.deleteUrlRedirections( + state.targetGuid, + chunk.map((c) => c.targetID) + ); + } catch (error: any) { + chunk.forEach((c) => failures.set(c, errorMessage(error))); + } + } + return failures; + } + + if (kind === "page") { + // Children before parents: a parent is only deletable once its children are gone. + const ordered = [...toDelete].sort((a, b) => (b.depth ?? 0) - (a.depth ?? 0)); + for (const candidate of ordered) { + try { + await deleteOne(candidate, opts); + } catch (error: any) { + failures.set(candidate, errorMessage(error)); + } + } + return failures; + } + + const limit = kind === "content" ? API_CONCURRENCY : 1; + await mapPool(toDelete, limit, async (candidate) => { + try { + await deleteOne(candidate, opts); + } catch (error: any) { + failures.set(candidate, errorMessage(error)); + } + }); + return failures; +} + +/** Drop the mapping records of deleted / unmapped candidates from one kind's mapping file. */ +function removeMappings(kind: DeleteKind, removed: DeleteCandidate[], opts: SyncDeletesOptions, locale?: string): void { + if (removed.length === 0 || state.preflight) return; + const fields = DELETE_KIND_ID_FIELDS[kind]; + const drop = new Set(removed.map((c) => `${c.sourceID}:${c.targetID}`)); + const records = readMappings(kind, opts, locale); + const kept = records.filter((r) => !drop.has(`${toId(r?.[fields.source])}:${toId(r?.[fields.target])}`)); + if (kept.length === records.length) return; + new fileOperations(opts.targetGuid).saveMappingFile(kept, DELETE_KIND_MAPPING_TYPE[kind], opts.sourceGuid, opts.targetGuid, locale ?? null); +} + +function preflightPhase(kind: DeleteKind): string { + return PHASE_LABEL[kind]; +} + +function recordPreflight(candidate: DeleteCandidate): void { + const base = { phase: preflightPhase(candidate.kind), name: candidate.name, locale: candidate.locale, detail: candidate.reason }; + switch (candidate.action) { + case "delete": + preflightReport.record({ ...base, action: "delete" }); + break; + case "conflict": + preflightReport.record({ ...base, action: "conflict" }); + break; + default: + preflightReport.record({ ...base, action: "skip" }); + } +} + +/** + * Run the delete phase. Called by the push orchestrator after every pusher, only with --syncDeletes. + */ +export async function runSyncDeletes(opts: SyncDeletesOptions): Promise { + const logger = getLoggerForGuid(opts.sourceGuid); + const result: SyncDeletesResult = { + successful: 0, + failed: 0, + skipped: 0, + status: "success", + failureDetails: [], + warningDetails: [], + candidates: [], + massDeleteRefused: false, + }; + + console.log(ansiColors.cyan("\n🗑 Sync deletes: checking for entities deleted in the source since the last sync...")); + + const { groups } = await planSyncDeletes(opts); + result.candidates = groups.reduce((all, g) => all.concat(g.candidates), []); + + const violations = findMassDeletes(groups); + if (violations.length > 0 && !state.allowMassDeletes) { + result.massDeleteRefused = true; + const lines = violations.map( + (v) => `${PHASE_LABEL[v.kind]}${v.locale ? ` [${v.locale}]` : ""}: ${v.deletes} of ${v.mapped} mapped entities` + ); + const message = + `Refusing to sync deletes: more than ${Math.round(MASS_DELETE_MAX_RATIO * 100)}% (and more than ${MASS_DELETE_MIN_COUNT}) ` + + `of the mapped entities would be deleted — ${lines.join("; ")}. This usually means a stale or partial local cache ` + + `(re-run with --fullPull). If the deletes are intended, re-run with --allowMassDeletes.`; + console.log(ansiColors.red(`\n❌ ${message}`)); + logger?.error(message); + result.failed = 1; + result.status = "error"; + result.failureDetails!.push({ name: "sync deletes", error: message }); + if (state.preflight) result.candidates.forEach(recordPreflight); + return result; + } + + for (const kind of DELETE_ORDER) { + for (const group of groups.filter((g) => g.kind === kind)) { + const { candidates, locale } = group; + if (candidates.length === 0) continue; + + if (state.preflight) { + candidates.forEach(recordPreflight); + candidates.forEach((c) => (c.action === "delete" ? result.successful++ : result.skipped++)); + continue; + } + + const failures = await executeGroup(candidates, opts); + const removed: DeleteCandidate[] = []; + + for (const candidate of candidates) { + const where = candidate.locale ? ` [${candidate.locale}]` : ""; + const label = `${PHASE_LABEL[kind].replace("Delete ", "").replace(/s$/, "")} ${candidate.name}${where}`; + const failure = failures.get(candidate); + if (candidate.action === "delete" && !failure) { + result.successful++; + removed.push(candidate); + logger?.success(`🗑 Deleted ${label} (target ID ${candidate.targetID}) — ${candidate.reason}`); + console.log(ansiColors.green(` ✓ deleted ${label}`)); + } else if (candidate.action === "delete") { + result.failed++; + result.status = "error"; + result.failureDetails!.push({ name: candidate.name, error: `delete failed: ${failure}`, type: FAILURE_TYPE[kind], locale: candidate.locale, guid: opts.targetGuid }); + logger?.error(`Failed to delete ${label} (target ID ${candidate.targetID}): ${failure}`); + console.log(ansiColors.red(` ✗ ${label}: delete failed — ${failure}`)); + } else if (candidate.action === "unmap") { + result.skipped++; + removed.push(candidate); + logger?.info(`${label}: ${candidate.reason}`); + } else if (candidate.action === "conflict" || candidate.action === "blocked") { + result.skipped++; + const detail: FailureDetail = { name: candidate.name, error: `not deleted — ${candidate.reason}`, type: FAILURE_TYPE[kind], locale: candidate.locale }; + result.warningDetails!.push(detail); + logger?.warning(`${label}: not deleted — ${candidate.reason}`); + console.log(ansiColors.yellow(` ⚠ ${label}: not deleted — ${candidate.reason}`)); + } else { + result.skipped++; + logger?.info(`${label}: ${candidate.reason}`); + } + } + + removeMappings(kind, removed, opts, locale); + } + } + + const planned = result.candidates.length; + console.log( + ansiColors.gray(`\nSync deletes: `) + + ansiColors.green(`${result.successful} deleted, `) + + ansiColors.yellow(`${result.skipped} skipped, `) + + (result.failed > 0 ? ansiColors.red : ansiColors.gray)(`${result.failed} failed`) + + ansiColors.gray(` (${planned} mapping(s) whose source is gone)\n`) + ); + + return result; +} diff --git a/src/lib/deleters/tests/delete-plan.test.ts b/src/lib/deleters/tests/delete-plan.test.ts new file mode 100644 index 0000000..d26916d --- /dev/null +++ b/src/lib/deleters/tests/delete-plan.test.ts @@ -0,0 +1,177 @@ +import { + collectFieldReferences, + decideAction, + DeleteCandidate, + findCandidates, + findMassDeletes, + indexSitemap, + mappingPairs, + modelFieldModelReferences, + pageZoneContentIDs, + pageZoneModuleNames, +} from "../delete-plan"; + +const facts = (over: Partial[0]> = {}) => ({ + sourceDeleted: true as const, + targetExists: true, + targetChanged: false, + ...over, +}); + +describe("decideAction", () => { + it("deletes when the source is deleted and the target is untouched", () => { + expect(decideAction(facts(), false).action).toBe("delete"); + }); + + it("keeps everything when the source delete is not confirmed", () => { + const result = decideAction(facts({ sourceDeleted: "unpublished in the source, not deleted" }), true); + expect(result).toEqual({ action: "keep", reason: "unpublished in the source, not deleted" }); + }); + + it("only drops the mapping when a live source still owns the target", () => { + const result = decideAction(facts({ sharedWithSourceID: 42, targetChanged: true }), true); + expect(result.action).toBe("unmap"); + expect(result.reason).toContain("42"); + }); + + it("only drops the mapping when the target is already gone", () => { + expect(decideAction(facts({ targetExists: false }), false).action).toBe("unmap"); + }); + + it("is a conflict when the target changed and --overwrite is off", () => { + expect(decideAction(facts({ targetChanged: true }), false).action).toBe("conflict"); + }); + + it("deletes a changed target under --overwrite", () => { + const result = decideAction(facts({ targetChanged: true }), true); + expect(result.action).toBe("delete"); + expect(result.reason).toContain("--overwrite"); + }); + + it("blocks before anything else that would delete, even under --overwrite", () => { + const result = decideAction(facts({ targetChanged: true, blockedBy: "still used by page home" }), true); + expect(result).toEqual({ action: "blocked", reason: "still used by page home" }); + }); +}); + +describe("mappingPairs / findCandidates", () => { + const records = [ + { sourceContentID: 1, targetContentID: 101 }, + { sourceContentID: 2, targetContentID: 102 }, + { sourceContentID: 3, targetContentID: 101 }, // dead duplicate onto a live target + { sourceContentID: null, targetContentID: 103 }, // unusable + { sourceContentID: "4", targetContentID: "104" }, // string ids still count + ]; + + it("reads each kind's id fields and drops unusable records", () => { + const pairs = mappingPairs("content", records); + expect(pairs.map((p) => [p.sourceID, p.targetID])).toEqual([ + [1, 101], + [2, 102], + [3, 101], + [4, 104], + ]); + }); + + it("returns records whose source is gone, and the targets still owned by a live source", () => { + const { candidates, liveTargets } = findCandidates(mappingPairs("content", records), new Set([1])); + expect(candidates.map((c) => c.sourceID)).toEqual([2, 3, 4]); + expect(liveTargets.get(101)).toBe(1); + expect(liveTargets.has(102)).toBe(false); + }); + + it("uses the guid-level field names", () => { + expect(mappingPairs("model", [{ sourceID: 5, targetID: 50 }])[0]).toMatchObject({ sourceID: 5, targetID: 50 }); + expect(mappingPairs("container", [{ sourceContentViewID: 6, targetContentViewID: 60 }])).toHaveLength(1); + expect(mappingPairs("gallery", [{ sourceMediaGroupingID: 7, targetMediaGroupingID: 70 }])).toHaveLength(1); + expect(mappingPairs("urlRedirection", [{ sourceUrlRedirectionID: 8, targetUrlRedirectionID: 80 }])).toHaveLength(1); + }); +}); + +describe("findMassDeletes", () => { + const candidates = (n: number, action: DeleteCandidate["action"] = "delete"): DeleteCandidate[] => + Array.from({ length: n }, (_, i) => ({ + kind: "content", + sourceID: i + 1, + targetID: i + 1001, + name: `item ${i}`, + record: {}, + action, + reason: "", + })); + + it("flags a kind losing more than half of many mapped entities", () => { + expect(findMassDeletes([{ kind: "content", locale: "en-us", mapped: 20, candidates: candidates(11) }])).toEqual([ + { kind: "content", locale: "en-us", deletes: 11, mapped: 20 }, + ]); + }); + + it("allows small deletes even when they are most of the kind", () => { + expect(findMassDeletes([{ kind: "model", mapped: 4, candidates: candidates(4) }])).toEqual([]); + }); + + it("allows many deletes when they are a minority", () => { + expect(findMassDeletes([{ kind: "content", mapped: 100, candidates: candidates(40) }])).toEqual([]); + }); + + it("counts only actual deletes", () => { + expect(findMassDeletes([{ kind: "content", mapped: 20, candidates: candidates(15, "conflict") }])).toEqual([]); + }); +}); + +describe("collectFieldReferences", () => { + it("finds linked content in every shape", () => { + const refs = collectFieldReferences({ + single: { contentid: 11, fulllist: false }, + list: { referencename: "Posts_List", sortids: "12,13" }, + upper: { ContentID: "14" }, + scalar: "15", + csv: "16, 17", + nested: [{ inner: { contentID: 18 } }], + text: "Hello 19 world", + }); + expect(Array.from(refs.contentIDs).sort((a, b) => a - b)).toEqual([11, 12, 13, 14, 15, 16, 17, 18]); + expect(Array.from(refs.containers)).toEqual(["posts_list"]); + }); + + it("finds galleries", () => { + expect(Array.from(collectFieldReferences({ photos: { galleryid: 3 } }).galleryIDs)).toEqual([3]); + }); + + it("tolerates missing fields", () => { + expect(collectFieldReferences(undefined).contentIDs.size).toBe(0); + }); +}); + +describe("page and model helpers", () => { + const page = { + zones: { + main: [ + { module: "RichText", item: { contentid: 5 } }, + { module: "Hero", item: { contentID: "6" } }, + ], + side: [{ module: "Nav", item: {} }], + }, + }; + + it("reads zone content and module names", () => { + expect(pageZoneContentIDs(page)).toEqual([5, 6]); + expect(pageZoneModuleNames(page)).toEqual(["richtext", "hero", "nav"]); + }); + + it("reads model-to-model field references", () => { + const model = { fields: [{ settings: { ContentDefinition: "Author" } }, { settings: {} }] }; + expect(modelFieldModelReferences(model)).toEqual(["author"]); + }); + + it("indexes a sitemap into parents and depths", () => { + const { parentOf, depthOf } = indexSitemap([ + { pages: [{ pageID: 1, childPages: [{ pageID: 2, childPages: [{ pageID: 3, childPages: [] }] }] }] }, + { pages: [{ pageID: 9, childPages: null }] }, + ]); + expect(parentOf.get(1)).toBeNull(); + expect(parentOf.get(3)).toBe(2); + expect(depthOf.get(3)).toBe(2); + expect(depthOf.get(9)).toBe(0); + }); +}); diff --git a/src/lib/deleters/tests/sync-deletes.test.ts b/src/lib/deleters/tests/sync-deletes.test.ts new file mode 100644 index 0000000..e7d2cc2 --- /dev/null +++ b/src/lib/deleters/tests/sync-deletes.test.ts @@ -0,0 +1,357 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { state, resetState, setState } from "../../../core/state"; +import { preflightReport } from "../../preflight/preflight-report"; +import { runSyncDeletes } from "../sync-deletes"; + +const SRC = "src-1"; +const TGT = "tgt-1"; +const LOCALE = "en-us"; + +function writeJson(file: string, data: any) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +function notFound(): any { + const error: any = new Error("Unable to retrieve"); + error.innerError = { response: { status: 404, data: "not found" } }; + return error; +} + +interface Fixture { + root: string; + deleted: string[]; + api: any; +} + +/** + * One source/target pair with a mapped entity of every kind, some deleted in the source. + * + * Source IDs are n, target IDs are 100 + n; target-only entities are 900+. + */ +function buildFixture(): Fixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "sync-deletes-")); + const src = (rel: string, data: any) => writeJson(path.join(root, SRC, rel), data); + const tgt = (rel: string, data: any) => writeJson(path.join(root, TGT, rel), data); + const map = (rel: string, data: any[]) => writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, rel, "mappings.json"), data); + + // ── source: only the live entities remain after the pull ── + src(`${LOCALE}/item/1.json`, { contentID: 1, properties: { state: 2, versionID: 1, referenceName: "posts" } }); + src(`${LOCALE}/page/10.json`, { pageID: 10, name: "home", properties: { state: 2, versionID: 1 }, zones: {} }); + src("models/20.json", { id: 20, referenceName: "Post", lastModifiedDate: "2026-01-01T00:00:00" }); + src("containers/30.json", { contentViewID: 30, referenceName: "posts", lastModifiedDate: "01/01/2026 10:00AM" }); + src("templates/40.json", { pageTemplateID: 40, pageTemplateName: "Main" }); + src("assets/json/1.json", { assetMedias: [{ mediaID: 50, fileName: "a.png", dateModified: "2026-01-01T00:00:00" }] }); + src("galleries/60.json", { mediaGroupingID: 60, name: "Live", modifiedOn: "01/01/2026 10:00AM" }); + src(`${LOCALE}/urlredirections/urlredirections.json`, { items: [{ id: 70, originUrl: "/live" }] }); + + // ── target ── + const item = (id: number, extra: any = {}) => ({ + contentID: id, + properties: { state: 2, versionID: 1, referenceName: "posts", definitionName: "Post" }, + fields: {}, + ...extra, + }); + [101, 102, 103, 104, 105].forEach((id) => tgt(`${LOCALE}/item/${id}.json`, item(id))); + // target-only item linking to 105 + tgt(`${LOCALE}/item/900.json`, item(900, { fields: { related: { contentid: 105 } } })); + [110, 111, 112, 113].forEach((id) => + tgt(`${LOCALE}/page/${id}.json`, { pageID: id, name: `page-${id}`, templateName: "Main", properties: { versionID: 1 }, zones: {} }) + ); + tgt(`${LOCALE}/page/950.json`, { pageID: 950, name: "native-child", templateName: "Main", properties: { versionID: 1 }, zones: {} }); + tgt(`${LOCALE}/page/960.json`, { pageID: 960, name: "native-page", templateName: "Old", properties: { versionID: 1 }, zones: {} }); + tgt("models/120.json", { id: 120, referenceName: "Post", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("models/121.json", { id: 121, referenceName: "Gone", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("models/122.json", { id: 122, referenceName: "InUse", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("containers/130.json", { contentViewID: 130, referenceName: "posts", contentDefinitionID: 120, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("containers/131.json", { contentViewID: 131, referenceName: "empty", contentDefinitionID: 121, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("containers/990.json", { contentViewID: 990, referenceName: "native", contentDefinitionID: 122, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("templates/140.json", { pageTemplateID: 140, pageTemplateName: "Main" }); + tgt("templates/141.json", { pageTemplateID: 141, pageTemplateName: "Old" }); + tgt("assets/json/1.json", { + assetMedias: [ + { mediaID: 150, fileName: "a.png", dateModified: "2026-01-01T00:00:00" }, + { mediaID: 151, fileName: "b.png", url: "https://cdn/b.png", dateModified: "2026-01-01T00:00:00" }, + ], + }); + tgt("galleries/160.json", { mediaGroupingID: 160, name: "Live", modifiedOn: "01/01/2026 10:00AM" }); + tgt("galleries/161.json", { mediaGroupingID: 161, name: "Gone", modifiedOn: "01/01/2026 10:00AM" }); + tgt(`${LOCALE}/urlredirections/urlredirections.json`, { items: [{ id: 170 }, { id: 171 }] }); + + // ── mappings ── + map(`${LOCALE}/item`, [1, 2, 3, 4, 5].map((n) => ({ sourceContentID: n, targetContentID: 100 + n, sourceVersionID: 1, targetVersionID: 1 }))); + map(`${LOCALE}/page`, [10, 11, 12, 13].map((n) => ({ sourcePageID: n, targetPageID: 100 + n, sourceVersionID: 1, targetVersionID: 1 }))); + map("models", [20, 21, 22].map((n) => ({ sourceID: n, targetID: 100 + n, targetLastModifiedDate: "2026-01-01T00:00:00" }))); + map("containers", [30, 31].map((n) => ({ sourceContentViewID: n, targetContentViewID: 100 + n, targetLastModifiedDate: "01/01/2026 10:00AM" }))); + map("templates", [40, 41].map((n) => ({ sourcePageTemplateID: n, targetPageTemplateID: 100 + n }))); + map("assets", [50, 51].map((n) => ({ sourceMediaID: n, targetMediaID: 100 + n, targetDateModified: "2026-01-01T00:00:00" }))); + map("galleries", [60, 61].map((n) => ({ sourceMediaGroupingID: n, targetMediaGroupingID: 100 + n, targetModifiedOn: "01/01/2026 10:00AM" }))); + map("urlredirections", [70, 71].map((n) => ({ sourceUrlRedirectionID: n, targetUrlRedirectionID: 100 + n, originUrl: `/r${n}` }))); + + const deleted: string[] = []; + const targetVersion: Record = { 104: 9 }; // 104 was edited on the target since the sync + const api = { + contentMethods: { + getContentItem: jest.fn(async (id: number, guid: string) => { + if (guid === SRC) { + if (id === 3) return { contentID: 3, properties: { state: 7 } }; + throw notFound(); + } + if (deleted.includes(`content:${id}`)) throw notFound(); + return { contentID: id, properties: { state: 2, versionID: targetVersion[id] ?? 1, referenceName: "posts" } }; + }), + deleteContent: jest.fn(async (id: number) => { + deleted.push(`content:${id}`); + return [1]; + }), + }, + pageMethods: { + getPage: jest.fn(async (id: number, guid: string) => { + if (guid === SRC) throw notFound(); + if (deleted.includes(`page:${id}`)) throw notFound(); + return { pageID: id, name: `page-${id}`, properties: { state: 2, versionID: 1 } }; + }), + getSitemap: jest.fn(async () => [ + { + pages: [ + { pageID: 110, childPages: [] }, + { pageID: 111, childPages: [{ pageID: 112, childPages: [] }] }, + { pageID: 113, childPages: [{ pageID: 950, childPages: [] }] }, + { pageID: 960, childPages: [] }, + ], + }, + ]), + deletePage: jest.fn(async (id: number) => { + deleted.push(`page:${id}`); + return [2]; + }), + deletePageTemplate: jest.fn(async (_guid: string, _locale: string, id: number) => { + deleted.push(`template:${id}`); + return "ok"; + }), + }, + urlRedirectionMethods: { + deleteUrlRedirections: jest.fn(async (_guid: string, ids: number[]) => { + ids.forEach((id) => deleted.push(`urlRedirection:${id}`)); + return { deleted: ids, notFound: [] }; + }), + }, + batchMethods: { getBatch: jest.fn(async () => ({ batchState: 3, items: [] })) }, + containerMethods: { + deleteContainer: jest.fn(async (id: number) => { + deleted.push(`container:${id}`); + return "ok"; + }), + }, + modelMethods: { + deleteModel: jest.fn(async (id: number) => { + deleted.push(`model:${id}`); + return "ok"; + }), + }, + assetMethods: { + deleteFile: jest.fn(async (id: number) => { + deleted.push(`asset:${id}`); + return "ok"; + }), + deleteGallery: jest.fn(async (_guid: string, id: number) => { + deleted.push(`gallery:${id}`); + return "ok"; + }), + }, + }; + + return { root, deleted, api }; +} + +const ALL_ELEMENTS = ["Galleries", "Assets", "Models", "Containers", "Content", "Templates", "Pages", "UrlRedirections"]; + +function configure(fixture: Fixture, extra: Record = {}) { + resetState(); + setState({ rootPath: fixture.root, sourceGuid: SRC, targetGuid: TGT, locales: LOCALE, isSync: true, syncDeletes: true, ...extra }); + state.cachedApiClient = fixture.api; + preflightReport.reset(); +} + +function run() { + return runSyncDeletes({ sourceGuid: SRC, targetGuid: TGT, locales: [LOCALE], elements: ALL_ELEMENTS }); +} + +function readMapping(fixture: Fixture, rel: string): any[] { + return JSON.parse(fs.readFileSync(path.join(fixture.root, "mappings", `${SRC}-${TGT}`, rel, "mappings.json"), "utf8")); +} + +function actions(result: Awaited>) { + const out: Record = {}; + result.candidates.forEach((c) => (out[`${c.kind}:${c.targetID}`] = c.action)); + return out; +} + +let logSpy: jest.SpyInstance; +beforeEach(() => { + logSpy = jest.spyOn(console, "log").mockImplementation(() => undefined); +}); +afterEach(() => logSpy.mockRestore()); + +describe("runSyncDeletes", () => { + it("plans every kind: deletes what is safe, keeps what is unpublished, changed or still used", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await run(); + + expect(actions(result)).toEqual({ + "urlRedirection:171": "delete", + "page:111": "delete", + "page:112": "delete", + "page:113": "blocked", // native child page 950 stays + "content:102": "delete", + "content:103": "keep", // unpublished in the source + "content:104": "conflict", // edited on the target + "content:105": "blocked", // linked from native item 900 + "container:131": "delete", + "template:141": "blocked", // native page 960 uses it + "model:121": "delete", + "model:122": "blocked", // native container 990 uses it + "asset:151": "delete", + "gallery:161": "delete", + }); + expect(result.candidates.find((c) => c.targetID === 103)?.reason).toContain("unpublished"); + expect(result.candidates.find((c) => c.targetID === 105)?.reason).toContain("content posts #900"); + }); + + it("deletes children before parents and removes only the deleted mappings", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await run(); + + expect(fixture.deleted).toEqual( + expect.arrayContaining(["page:112", "page:111", "content:102", "container:131", "model:121", "asset:151", "gallery:161"]) + ); + expect(fixture.deleted.indexOf("page:112")).toBeLessThan(fixture.deleted.indexOf("page:111")); + expect(fixture.deleted).not.toContain("content:104"); + expect(fixture.deleted).not.toContain("template:141"); + expect(fixture.api.urlRedirectionMethods.deleteUrlRedirections).toHaveBeenCalledWith(TGT, [171]); + + expect(readMapping(fixture, `${LOCALE}/item`).map((r) => r.targetContentID)).toEqual([101, 103, 104, 105]); + expect(readMapping(fixture, `${LOCALE}/page`).map((r) => r.targetPageID)).toEqual([110, 113]); + expect(readMapping(fixture, "models").map((r) => r.targetID)).toEqual([120, 122]); + expect(readMapping(fixture, "urlredirections").map((r) => r.targetUrlRedirectionID)).toEqual([170]); + + expect(result.successful).toBe(8); + expect(result.failed).toBe(0); + expect(result.warningDetails!.map((w) => w.name).sort()).toEqual(["InUse", "Old", "page-113", "posts #104", "posts #105"].sort()); + }); + + it("deletes a target edited since the sync only with --overwrite", async () => { + const fixture = buildFixture(); + configure(fixture, { overwrite: true }); + const result = await run(); + expect(actions(result)["content:104"]).toBe("delete"); + expect(fixture.deleted).toContain("content:104"); + }); + + it("writes nothing under --preflight and reports the plan", async () => { + const fixture = buildFixture(); + configure(fixture, { preflight: true }); + const before = readMapping(fixture, `${LOCALE}/item`); + await run(); + + expect(fixture.deleted).toEqual([]); + expect(fixture.api.urlRedirectionMethods.deleteUrlRedirections).not.toHaveBeenCalled(); + expect(readMapping(fixture, `${LOCALE}/item`)).toEqual(before); + const totals = preflightReport.getTotals(); + expect(totals.delete).toBe(8); + expect(totals.conflict).toBe(1); + expect(preflightReport.getEntries().find((e) => e.name === "posts #102")).toMatchObject({ + phase: "Delete Content", + action: "delete", + locale: LOCALE, + }); + }); + + it("counts a target that disappeared before its delete (e.g. with its page) as done", async () => { + const fixture = buildFixture(); + configure(fixture); + fixture.api.pageMethods.deletePage.mockImplementation(async (id: number) => { + fixture.deleted.push(`page:${id}`); + if (id === 111) fixture.deleted.push("content:102"); // page delete took its zone content along + return [2]; + }); + const result = await run(); + expect(fixture.api.contentMethods.deleteContent).not.toHaveBeenCalledWith(102, expect.anything(), expect.anything(), expect.anything(), expect.anything()); + expect(result.failed).toBe(0); + expect(readMapping(fixture, `${LOCALE}/item`).map((r) => r.targetContentID)).not.toContain(102); + }); + + it("reports a failed delete and keeps its mapping", async () => { + const fixture = buildFixture(); + configure(fixture); + fixture.api.modelMethods.deleteModel.mockRejectedValue(new Error("model in use")); + const result = await run(); + expect(result.failed).toBe(1); + expect(result.status).toBe("error"); + expect(result.failureDetails![0]).toMatchObject({ name: "Gone", type: "model" }); + expect(readMapping(fixture, "models").map((r) => r.targetID)).toContain(121); + }); + + it("does not delete anything of a kind whose source list is empty", async () => { + const fixture = buildFixture(); + configure(fixture); + fs.rmSync(path.join(fixture.root, SRC, "models"), { recursive: true }); + const result = await run(); + expect(actions(result)["model:121"]).toBe("keep"); + expect(fixture.deleted).not.toContain("model:121"); + }); + + it("only considers the kinds in --elements", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await runSyncDeletes({ sourceGuid: SRC, targetGuid: TGT, locales: [LOCALE], elements: ["Content"] }); + expect(new Set(result.candidates.map((c) => c.kind))).toEqual(new Set(["content"])); + }); +}); + +describe("mass-delete guard", () => { + function massFixture(): Fixture { + const fixture = buildFixture(); + const records = Array.from({ length: 12 }, (_, i) => ({ + sourceContentID: 200 + i, + targetContentID: 300 + i, + sourceVersionID: 1, + targetVersionID: 1, + })); + records.forEach((r) => + writeJson(path.join(fixture.root, TGT, LOCALE, "item", `${r.targetContentID}.json`), { + contentID: r.targetContentID, + properties: { state: 2, versionID: 1, referenceName: "posts" }, + fields: {}, + }) + ); + writeJson(path.join(fixture.root, "mappings", `${SRC}-${TGT}`, LOCALE, "item", "mappings.json"), [ + { sourceContentID: 1, targetContentID: 101, sourceVersionID: 1, targetVersionID: 1 }, + ...records, + ]); + return fixture; + } + + it("refuses to delete anything when one kind would lose most of its mapped entities", async () => { + const fixture = massFixture(); + configure(fixture); + const result = await run(); + expect(result.massDeleteRefused).toBe(true); + expect(result.failed).toBe(1); + expect(fixture.deleted).toEqual([]); + expect(result.failureDetails![0].error).toContain("--allowMassDeletes"); + }); + + it("goes ahead with --allowMassDeletes", async () => { + const fixture = massFixture(); + configure(fixture, { allowMassDeletes: true }); + const result = await run(); + expect(result.massDeleteRefused).toBe(false); + expect(fixture.deleted.filter((d) => d.startsWith("content:3"))).toHaveLength(12); + }); +}); diff --git a/src/lib/list-order/sync-list-order.ts b/src/lib/list-order/sync-list-order.ts new file mode 100644 index 0000000..21c56ef --- /dev/null +++ b/src/lib/list-order/sync-list-order.ts @@ -0,0 +1,267 @@ +/** + * List order (PROD-2992): put each synced content list on the target in the source's item order. + * + * The content save can't carry order across: an update keeps the target's own itemOrder, a new item + * is appended when its source position is already taken, and a reorder alone in the source makes no + * new version, so change detection never sees it. This phase runs after content (and --syncDeletes) + * and sets the order of each list that differs through the Management API's list item-order + * endpoint, which renumbers the staging and the live list in place, with no new versions. + * + * Planning is pure (planListOrders) and works from the local pulls plus the mapping files, so + * preflight can report what would change. Only lists whose expected order differs from the target's + * pulled order are sent, and lists that gained an item this run (it isn't in the pull yet). A mapped + * item missing from the pull that this run didn't save is a stale mapping, not a new item: it doesn't + * make the list differ, or every run would send that list again. + */ + +import ansiColors from "ansi-colors"; +import { state, getApiClient, getLoggerForGuid } from "../../core/state"; +import { fileOperations } from "../../core/fileOperations"; +import { GuidDataLoader } from "../pushers/guid-data-loader"; +import { preflightReport } from "../preflight/preflight-report"; +import { FailureDetail, PusherResult } from "../../types/sourceData"; + +export interface ListOrderOptions { + sourceGuid: string; + targetGuid: string; + locales: string[]; +} + +/** The minimum of a pulled content item this phase reads. */ +export interface ListItem { + contentID: number; + properties?: { referenceName?: string; itemOrder?: number | null; state?: number }; +} + +/** One list whose target order has to change. */ +export interface ListOrderPlan { + locale: string; + sourceReferenceName: string; + targetReferenceName: string; + /** Target content IDs in the source's order. Target-only items are not listed; the API keeps them after these. */ + contentIDs: number[]; +} + +const DELETED_STATE = 3; + +const lower = (value: string | undefined | null) => (value || "").toLowerCase(); + +/** List order as the server reads it: by itemOrder (missing last), then content ID. */ +function sortByListOrder(items: T[]): T[] { + return [...items].sort( + (a, b) => + (a.properties?.itemOrder ?? Number.MAX_SAFE_INTEGER) - (b.properties?.itemOrder ?? Number.MAX_SAFE_INTEGER) || + a.contentID - b.contentID + ); +} + +/** Whether every value is set and strictly increasing: otherwise the order isn't defined and the server renumbers. */ +function hasDefinedOrder(sorted: ListItem[]): boolean { + return sorted.every( + (item, i) => + item.properties?.itemOrder != null && + (i === 0 || (item.properties.itemOrder as number) > (sorted[i - 1].properties?.itemOrder as number)) + ); +} + +function groupByList(items: ListItem[]): Map { + const lists = new Map(); + for (const item of items) { + if (!item?.contentID || item.properties?.state === DELETED_STATE) continue; + const key = lower(item.properties?.referenceName); + if (!key) continue; + if (!lists.has(key)) lists.set(key, []); + lists.get(key)!.push(item); + } + return lists; +} + +/** + * Work out which lists need a new order on the target, for one locale. + * + * @param sourceItems every content item in the source pull for the locale (not just a scoped run's) + * @param targetItems every content item in the target pull for the locale + * @param contentMap source content ID -> target content ID, from the locale's item mappings + * @param containerMap source list reference name (lower case) -> target reference name, from the container mappings + * @param scope when set, only these source lists (lower case) are considered, e.g. a scoped run's + * @param savedThisRun target content IDs this run saved; a mapped item missing from the target pull + * only counts as new when it's one of these + */ +export function planListOrders( + locale: string, + sourceItems: ListItem[], + targetItems: ListItem[], + contentMap: Map, + containerMap: Map, + scope?: Set, + savedThisRun: Set = new Set() +): ListOrderPlan[] { + const sourceLists = groupByList(sourceItems); + const targetLists = groupByList(targetItems); + const plans: ListOrderPlan[] = []; + + for (const [sourceKey, items] of Array.from(sourceLists.entries())) { + if (scope && !scope.has(sourceKey)) continue; + + const targetReferenceName = containerMap.get(sourceKey); + if (!targetReferenceName) continue; + + // Source order, mapped. Two source items mapped to one target keep the first position. + const contentIDs: number[] = []; + for (const item of sortByListOrder(items)) { + const targetID = contentMap.get(item.contentID); + if (targetID && !contentIDs.includes(targetID)) contentIDs.push(targetID); + } + if (contentIDs.length === 0) continue; + + // What the API would produce: the listed items, then the target's others in their current order. + const current = sortByListOrder(targetLists.get(lower(targetReferenceName)) || []); + const currentIDs = current.map((i) => i.contentID); + const listed = new Set(contentIDs); + const expected = [...contentIDs.filter((id) => currentIDs.includes(id)), ...currentIDs.filter((id) => !listed.has(id))]; + + const added = contentIDs.some((id) => !currentIDs.includes(id) && savedThisRun.has(id)); + const differs = expected.some((id, i) => id !== currentIDs[i]); + if (!added && !differs && hasDefinedOrder(current)) continue; + + plans.push({ locale, sourceReferenceName: items[0].properties!.referenceName!, targetReferenceName, contentIDs }); + } + + return plans.sort((a, b) => lower(a.targetReferenceName).localeCompare(lower(b.targetReferenceName))); +} + +/** source content ID -> target content ID. The last record for a source wins, as in the content mapper. */ +export function contentMapFrom(records: any[]): Map { + const map = new Map(); + for (const r of records || []) { + const source = Number(r?.sourceContentID); + const target = Number(r?.targetContentID); + if (source > 0 && target > 0) map.set(source, target); + } + return map; +} + +/** source list reference name (lower case) -> target reference name. */ +export function containerMapFrom(records: any[], targetLists: string[] = []): Map { + const map = new Map(); + for (const r of records || []) { + if (r?.sourceReferenceName && r?.targetReferenceName) map.set(lower(r.sourceReferenceName), r.targetReferenceName); + } + // A list with the same reference name on both sides but no mapping record (e.g. synced before + // container mappings were kept) still counts as the same list. + for (const name of targetLists) { + if (!map.has(lower(name))) map.set(lower(name), name); + } + return map; +} + +/** + * Run the list order phase. Called by the push orchestrator after content (and --syncDeletes). + * + * @param scopedContent per locale, the source content a scoped run (--pages/--containers/--models) + * pushed; only their lists are ordered. Omit for a full run. + * @param savedContentIDs per locale, the target content IDs this run saved. + */ +export async function runSyncListOrder( + opts: ListOrderOptions, + scopedContent?: Map, + savedContentIDs?: Map +): Promise { + const logger = getLoggerForGuid(opts.sourceGuid); + const result: PusherResult = { successful: 0, failed: 0, skipped: 0, status: "success", failureDetails: [], warningDetails: [] }; + const fileOps = new fileOperations(opts.targetGuid); + const containerRecords = fileOps.getMappingFile("containers", opts.sourceGuid, opts.targetGuid, null); + + console.log(ansiColors.cyan("\n↕ List order: checking synced lists against the source's item order...")); + + const sourceLoader = new GuidDataLoader(opts.sourceGuid); + const targetLoader = new GuidDataLoader(opts.targetGuid); + const plans: ListOrderPlan[] = []; + + for (const locale of opts.locales) { + const source = await sourceLoader.loadCompleteGuidEntities(locale); + const target = await targetLoader.loadCompleteGuidEntities(locale); + const itemRecords = fileOps.getMappingFile("item", opts.sourceGuid, opts.targetGuid, locale); + const targetLists = Array.from(new Set((target.content || []).map((i: any) => i?.properties?.referenceName).filter(Boolean))); + const scoped = scopedContent?.get(locale); + const scope = scoped ? new Set(scoped.map((i) => lower(i.properties?.referenceName)).filter(Boolean)) : undefined; + + plans.push( + ...planListOrders( + locale, + source.content || [], + target.content || [], + contentMapFrom(Array.isArray(itemRecords) ? itemRecords : []), + containerMapFrom(Array.isArray(containerRecords) ? containerRecords : [], targetLists as string[]), + scope, + new Set(savedContentIDs?.get(locale) || []) + ) + ); + } + + if (state.preflight) { + for (const plan of plans) { + preflightReport.record({ + phase: "List order", + action: "update", + name: plan.targetReferenceName, + locale: plan.locale, + detail: `put ${plan.contentIDs.length} item(s) in the source's order`, + }); + result.successful++; + } + console.log(ansiColors.gray(`List order: ${plans.length} list(s) would be reordered\n`)); + return result; + } + + const api = getApiClient(); + for (const plan of plans) { + const label = `${plan.targetReferenceName} [${plan.locale}]`; + try { + const response: any = await api.contentMethods.setContentListItemOrder( + plan.targetReferenceName, + opts.targetGuid, + plan.locale, + plan.contentIDs, + true + ); + const skippedIDs: number[] = response?.skippedContentIDs || []; + if (skippedIDs.length > 0) { + const detail: FailureDetail = { + name: plan.targetReferenceName, + error: `list order: ${skippedIDs.length} mapped item(s) aren't in the target list and were left out (target IDs ${skippedIDs.join(", ")})`, + type: "content", + locale: plan.locale, + guid: opts.targetGuid, + }; + result.warningDetails!.push(detail); + logger?.warning(`${label}: ${detail.error}`); + } + if ((response?.itemsUpdated ?? 0) > 0) { + result.successful++; + logger?.success(`↕ Reordered ${label}: ${response.itemsUpdated} item(s) moved (${(response.updatedContentIDs || []).join(", ")})`); + console.log(ansiColors.green(` ✓ reordered ${label}: ${response.itemsUpdated} item(s) moved`)); + } else { + result.skipped++; + logger?.info(`${label}: already in the source's order`); + } + } catch (error: any) { + result.failed++; + result.status = "error"; + const message = error?.message || String(error); + result.failureDetails!.push({ name: plan.targetReferenceName, error: `list order failed: ${message}`, type: "content", locale: plan.locale, guid: opts.targetGuid }); + logger?.error(`Failed to set the order of ${label}: ${message}`); + console.log(ansiColors.red(` ✗ ${label}: list order failed — ${message}`)); + } + } + + console.log( + ansiColors.gray(`List order: `) + + ansiColors.green(`${result.successful} reordered, `) + + ansiColors.yellow(`${result.skipped} already in order, `) + + (result.failed > 0 ? ansiColors.red : ansiColors.gray)(`${result.failed} failed`) + + ansiColors.gray(` (${plans.length} list(s) differed from the source)\n`) + ); + + return result; +} diff --git a/src/lib/list-order/tests/sync-list-order.test.ts b/src/lib/list-order/tests/sync-list-order.test.ts new file mode 100644 index 0000000..6c932c6 --- /dev/null +++ b/src/lib/list-order/tests/sync-list-order.test.ts @@ -0,0 +1,294 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { state, resetState, setState } from "../../../core/state"; +import { preflightReport } from "../../preflight/preflight-report"; +import { ListItem, containerMapFrom, contentMapFrom, planListOrders, runSyncListOrder } from "../sync-list-order"; + +const LOCALE = "en-us"; + +// contentID, itemOrder (list "links" unless given) +const item = (contentID: number, itemOrder: number | null, referenceName = "links", state = 2): ListItem => ({ + contentID, + properties: { referenceName, itemOrder, state }, +}); + +// Source n maps to target 100 + n. +const contentMap = (...ids: number[]) => new Map(ids.map((n) => [n, 100 + n] as [number, number])); +const containerMap = new Map([["links", "links"]]); + +describe("planListOrders", () => { + it("sends the source order, mapped, when the target is in another order", () => { + const plans = planListOrders( + LOCALE, + [item(1, 0), item(2, 1), item(3, 2)], + [item(101, 0), item(103, 1), item(102, 2)], + contentMap(1, 2, 3), + containerMap + ); + + expect(plans).toEqual([{ locale: LOCALE, sourceReferenceName: "links", targetReferenceName: "links", contentIDs: [101, 102, 103] }]); + }); + + it("leaves a list alone when the target already sorts that way, gaps and all", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1)], [item(101, 4), item(102, 20)], contentMap(1, 2), containerMap); + + expect(plans).toEqual([]); + }); + + it("reads order by value, not by position in the pull", () => { + const plans = planListOrders(LOCALE, [item(2, 1), item(1, 0)], [item(102, 1), item(101, 0)], contentMap(1, 2), containerMap); + + expect(plans).toEqual([]); + }); + + it("includes a list that gained an item this run (not in the target pull yet)", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1)], [item(102, 0)], contentMap(1, 2), containerMap, undefined, new Set([101])); + + expect(plans[0].contentIDs).toEqual([101, 102]); + }); + + it("ignores a stale mapping: a mapped item this run didn't save and the target doesn't have", () => { + // 101 is mapped but gone from the target; 102 and 103 are already in order. + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1), item(3, 2)], [item(102, 0), item(103, 1)], contentMap(1, 2, 3), containerMap); + + expect(plans).toEqual([]); + }); + + it("still reorders around a stale mapping when the rest is out of order", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1), item(3, 2)], [item(103, 0), item(102, 1)], contentMap(1, 2, 3), containerMap); + + expect(plans[0].contentIDs).toEqual([101, 102, 103]); + }); + + it("counts target-only items: in order when they already sit after the synced ones", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1)], [item(101, 0), item(102, 1), item(900, 2)], contentMap(1, 2), containerMap); + + expect(plans).toEqual([]); + }); + + it("reorders when a target-only item sits above a synced one, and doesn't list the target-only item", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1)], [item(900, 0), item(101, 1), item(102, 2)], contentMap(1, 2), containerMap); + + expect(plans[0].contentIDs).toEqual([101, 102]); + }); + + it("treats tied target values as out of order", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1)], [item(101, 3), item(102, 3)], contentMap(1, 2), containerMap); + + expect(plans).toHaveLength(1); + }); + + it("sorts missing source values last, then by content ID", () => { + const plans = planListOrders( + LOCALE, + [item(3, null), item(2, 0), item(1, null)], + [], + contentMap(1, 2, 3), + containerMap, + undefined, + new Set([101, 102, 103]) + ); + + expect(plans[0].contentIDs).toEqual([102, 101, 103]); + }); + + it("leaves out source items with no mapping, and skips a list with none mapped", () => { + const plans = planListOrders( + LOCALE, + [item(1, 0), item(2, 1), item(5, 0, "other")], + [item(102, 0)], + contentMap(2), + new Map([["links", "links"], ["other", "other"]]) + ); + + expect(plans).toEqual([]); + }); + + it("keeps the first position when two source items map to one target item", () => { + const plans = planListOrders(LOCALE, [item(1, 0), item(2, 1), item(3, 2)], [], new Map([[1, 101], [2, 103], [3, 103]]), containerMap, undefined, new Set([101, 103])); + + expect(plans[0].contentIDs).toEqual([101, 103]); + }); + + it("ignores deleted items on both sides", () => { + const plans = planListOrders( + LOCALE, + [item(1, 0), item(9, 1, "links", 3), item(2, 2)], + [item(101, 0), item(102, 1), item(109, 2, "links", 3)], + contentMap(1, 2, 9), + containerMap + ); + + expect(plans).toEqual([]); + }); + + it("maps the list through the container mapping, case-insensitively", () => { + const plans = planListOrders( + LOCALE, + [item(1, 0, "FooterLinks"), item(2, 1, "FooterLinks")], + [item(102, 0, "footerlinks_t"), item(101, 1, "footerlinks_t")], + contentMap(1, 2), + new Map([["footerlinks", "FooterLinks_T"]]) + ); + + expect(plans).toEqual([ + { locale: LOCALE, sourceReferenceName: "FooterLinks", targetReferenceName: "FooterLinks_T", contentIDs: [101, 102] }, + ]); + }); + + it("skips a list with no container mapping", () => { + expect(planListOrders(LOCALE, [item(1, 0), item(2, 1)], [], contentMap(1, 2), new Map())).toEqual([]); + }); + + it("only considers the scoped lists, but sends every mapped item in them", () => { + const plans = planListOrders( + LOCALE, + [item(1, 0), item(2, 1), item(3, 0, "other"), item(4, 1, "other")], + [item(102, 0), item(101, 1), item(104, 0, "other"), item(103, 1, "other")], + contentMap(1, 2, 3, 4), + new Map([["links", "links"], ["other", "other"]]), + new Set(["links"]) + ); + + expect(plans.map((p) => [p.targetReferenceName, p.contentIDs])).toEqual([["links", [101, 102]]]); + }); +}); + +describe("mapping helpers", () => { + it("contentMapFrom lets the last record for a source win and drops incomplete ones", () => { + const map = contentMapFrom([ + { sourceContentID: 1, targetContentID: 101 }, + { sourceContentID: 1, targetContentID: 111 }, + { sourceContentID: 2, targetContentID: -1 }, + ]); + + expect(Array.from(map.entries())).toEqual([[1, 111]]); + }); + + it("containerMapFrom keys by lower-case source name and falls back to same-named target lists", () => { + const map = containerMapFrom([{ sourceReferenceName: "Posts", targetReferenceName: "Posts_T" }], ["Links", "posts"]); + + expect(map.get("posts")).toBe("Posts_T"); + expect(map.get("links")).toBe("Links"); + }); +}); + +describe("runSyncListOrder", () => { + const SRC = "src-1"; + const TGT = "tgt-1"; + let root: string; + let api: any; + let logSpy: jest.SpyInstance; + + const writeJson = (rel: string, data: any) => { + const file = path.join(root, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); + }; + + beforeEach(() => { + logSpy = jest.spyOn(console, "log").mockImplementation(() => {}); + root = fs.mkdtempSync(path.join(os.tmpdir(), "list-order-")); + + // Source: links in order 1, 2, 3. Target: 103, 101, 102, plus a target-only 900 on top. + [1, 2, 3].forEach((n, i) => + writeJson(`${SRC}/${LOCALE}/item/${n}.json`, { contentID: n, properties: { state: 2, referenceName: "links", itemOrder: i } }) + ); + [[900, 0], [103, 1], [101, 2], [102, 3]].forEach(([id, order]) => + writeJson(`${TGT}/${LOCALE}/item/${id}.json`, { contentID: id, properties: { state: 2, referenceName: "links", itemOrder: order } }) + ); + writeJson(`mappings/${SRC}-${TGT}/${LOCALE}/item/mappings.json`, [1, 2, 3].map((n) => ({ sourceContentID: n, targetContentID: 100 + n }))); + writeJson(`mappings/${SRC}-${TGT}/containers/mappings.json`, [{ sourceReferenceName: "links", targetReferenceName: "links" }]); + + api = { + contentMethods: { + setContentListItemOrder: jest.fn(async (_ref: string, _guid: string, _locale: string, ids: number[]) => ({ + contentIDs: [...ids, 900], + itemsUpdated: 4, + updatedContentIDs: [...ids, 900], + skippedContentIDs: [], + })), + }, + }; + + resetState(); + setState({ rootPath: root, sourceGuid: SRC, targetGuid: TGT, locales: LOCALE, isSync: true }); + state.cachedApiClient = api; + preflightReport.reset(); + }); + + afterEach(() => { + logSpy.mockRestore(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + const run = (scoped?: Map) => runSyncListOrder({ sourceGuid: SRC, targetGuid: TGT, locales: [LOCALE] }, scoped); + + it("sets the target list to the source order, skipping unknown IDs, and counts it", async () => { + const result = await run(); + + expect(api.contentMethods.setContentListItemOrder).toHaveBeenCalledWith("links", TGT, LOCALE, [101, 102, 103], true); + expect(result).toMatchObject({ successful: 1, failed: 0, skipped: 0, status: "success" }); + }); + + it("counts a list the API says was already in order as skipped", async () => { + api.contentMethods.setContentListItemOrder.mockResolvedValueOnce({ contentIDs: [101, 102, 103, 900], itemsUpdated: 0, updatedContentIDs: [], skippedContentIDs: [] }); + + const result = await run(); + + expect(result).toMatchObject({ successful: 0, skipped: 1, failed: 0 }); + }); + + it("warns about mapped items the target list doesn't have", async () => { + api.contentMethods.setContentListItemOrder.mockResolvedValueOnce({ contentIDs: [101, 102, 900], itemsUpdated: 2, updatedContentIDs: [101, 102], skippedContentIDs: [103] }); + + const result = await run(); + + expect(result.successful).toBe(1); + expect(result.warningDetails).toHaveLength(1); + expect(result.warningDetails![0]).toMatchObject({ name: "links", locale: LOCALE }); + expect(result.warningDetails![0].error).toContain("103"); + }); + + it("records a failure and carries on when the API refuses", async () => { + api.contentMethods.setContentListItemOrder.mockRejectedValueOnce(new Error("Unable to set the item order (HTTP 404): not found")); + + const result = await run(); + + expect(result).toMatchObject({ failed: 1, status: "error" }); + expect(result.failureDetails![0].error).toContain("HTTP 404"); + }); + + it("calls nothing when every list is already in the source order", async () => { + writeJson(`${TGT}/${LOCALE}/item/900.json`, { contentID: 900, properties: { state: 2, referenceName: "links", itemOrder: 9 } }); + [[101, 0], [102, 1], [103, 2]].forEach(([id, order]) => + writeJson(`${TGT}/${LOCALE}/item/${id}.json`, { contentID: id, properties: { state: 2, referenceName: "links", itemOrder: order } }) + ); + + const result = await run(); + + expect(api.contentMethods.setContentListItemOrder).not.toHaveBeenCalled(); + expect(result).toMatchObject({ successful: 0, skipped: 0, failed: 0 }); + }); + + it("preflight records the list and calls nothing", async () => { + state.preflight = true; + + const result = await run(); + + expect(api.contentMethods.setContentListItemOrder).not.toHaveBeenCalled(); + expect(result.successful).toBe(1); + expect(preflightReport.toJSON().entries).toEqual( + expect.arrayContaining([expect.objectContaining({ phase: "List order", action: "update", name: "links", locale: LOCALE })]) + ); + }); + + it("a scoped run only orders the lists its content was in", async () => { + const result = await run(new Map([[LOCALE, [item(1, 0, "other")]]])); + + expect(api.contentMethods.setContentListItemOrder).not.toHaveBeenCalled(); + expect(result.successful).toBe(0); + }); +}); diff --git a/src/lib/mappers/content-item-mapper.ts b/src/lib/mappers/content-item-mapper.ts index 6d64815..f7258c0 100644 --- a/src/lib/mappers/content-item-mapper.ts +++ b/src/lib/mappers/content-item-mapper.ts @@ -159,6 +159,18 @@ export class ContentItemMapper { this.saveMapping(); } + /** + * PROD-3076: make the next sync treat this item's source as changed, so it is pushed again. Used + * when an item was created without its references (pass 1 of a reference cycle) and the update + * that fills them in failed. + */ + markSourceChanged(sourceContentID: number): void { + const mapping = this.getContentItemMappingByContentID(sourceContentID, "source"); + if (!mapping) return; + mapping.sourceVersionID = 0; + this.saveMapping(); + } + updateMapping( sourceContentItem: mgmtApi.ContentItem, targetContentItem: mgmtApi.ContentItem, diff --git a/src/lib/mappers/refresh-container-dates.ts b/src/lib/mappers/refresh-container-dates.ts new file mode 100644 index 0000000..75f110b --- /dev/null +++ b/src/lib/mappers/refresh-container-dates.ts @@ -0,0 +1,99 @@ +/** + * PROD-3042: after a sync saves pages, advance the container mappings the page saves made look + * "changed on the target". + * + * Saving a page (UpdatePageItem) aligns the "requires approval" setting of the non-shared + * containers of its components with the page's, and stamps lastModifiedDate on each one it changes. + * The container pusher compares a target container's lastModifiedDate with the one in its mapping, + * so on the next run those containers looked edited on the target, and every following sync needed + * --overwrite. That's the same family as PROD-2604 (models) and PROD-2628 (pages in other locales). + * + * A container that was in sync with its mapping when the run started, and has a newer date now, + * was changed by this run, so its mapping takes the new date. A container already changed on the + * target before the run is left alone, so a real target edit still shows as a conflict. + */ + +import { parse } from "date-fns"; +import { state, getApiClient, getLoggerForGuid } from "../../core/state"; +import { fileOperations } from "../../core/fileOperations"; +import { GuidDataLoader } from "../pushers/guid-data-loader"; + +const DATE_FORMAT = "MM/dd/yyyy hh:mma"; + +function toTime(value: string | undefined | null): number | undefined { + if (!value) return undefined; + const time = parse(value, DATE_FORMAT, new Date()).getTime(); + return Number.isNaN(time) ? undefined : time; +} + +export interface ContainerDateUpdate { + targetContentViewID: number; + from: string; + to: string; +} + +/** + * Work out which container mapping records to advance, and advance them in place. + * + * @param records container mapping records (targetContentViewID, targetLastModifiedDate) + * @param pulledDates target container ID -> lastModifiedDate in the target pull, i.e. at the start of the run + * @param currentDates target container ID -> lastModifiedDate now, after the run's writes + */ +export function advanceContainerDates( + records: any[], + pulledDates: Map, + currentDates: Map +): ContainerDateUpdate[] { + const updates: ContainerDateUpdate[] = []; + for (const record of records || []) { + const id = Number(record?.targetContentViewID); + const mapped = toTime(record?.targetLastModifiedDate); + const current = currentDates.get(id); + const currentTime = toTime(current); + if (!(id > 0) || mapped === undefined || currentTime === undefined || currentTime <= mapped) continue; + + // Not in the pull (e.g. created this run): its mapping was written by this run. + const pulled = toTime(pulledDates.get(id)); + const inSyncAtStart = pulled === undefined || pulled <= mapped; + if (!inSyncAtStart) continue; + + updates.push({ targetContentViewID: id, from: record.targetLastModifiedDate, to: current! }); + record.targetLastModifiedDate = current; + } + return updates; +} + +/** Re-read the target's containers and advance the mappings this run's page saves made stale. */ +export async function refreshContainerDatesAfterPageSaves(sourceGuid: string, targetGuid: string, locale: string): Promise { + if (state.preflight) return 0; + const logger = getLoggerForGuid(sourceGuid); + const fileOps = new fileOperations(targetGuid); + const records = fileOps.getMappingFile("containers", sourceGuid, targetGuid, null); + if (!Array.isArray(records) || records.length === 0) return 0; + + try { + const pulled = await new GuidDataLoader(targetGuid).loadCompleteGuidEntities(locale); + const pulledDates = new Map( + (pulled.containers || []).map((c: any) => [Number(c.contentViewID), c.lastModifiedDate] as [number, string]) + ); + const current = await getApiClient().containerMethods.getContainerList(targetGuid); + const currentDates = new Map( + (current || []).map((c: any) => [Number(c.contentViewID), c.lastModifiedDate] as [number, string]) + ); + + const updates = advanceContainerDates(records, pulledDates, currentDates); + if (updates.length > 0) { + fileOps.saveMappingFile(records, "containers", sourceGuid, targetGuid); + logger?.info( + `Container mappings: ${updates.length} advanced to the date this sync's page saves set (` + + updates.map((u) => u.targetContentViewID).join(", ") + + ")" + ); + } + return updates.length; + } catch (error: any) { + // Bookkeeping only: the sync itself succeeded. The cost of failing here is a false conflict next run. + logger?.warning(`Could not refresh container mapping dates after page saves: ${error?.message || error}`); + return 0; + } +} diff --git a/src/lib/mappers/tests/refresh-container-dates.test.ts b/src/lib/mappers/tests/refresh-container-dates.test.ts new file mode 100644 index 0000000..fd7484c --- /dev/null +++ b/src/lib/mappers/tests/refresh-container-dates.test.ts @@ -0,0 +1,113 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { state, resetState, setState } from "../../../core/state"; +import { advanceContainerDates, refreshContainerDatesAfterPageSaves } from "../refresh-container-dates"; + +const T0 = "10/30/2026 10:00AM"; // recorded at the last sync +const T1 = "10/30/2026 10:05AM"; // set by this run's page save +const TX = "10/30/2026 09:30AM"; // older + +const record = (id: number, date: string) => ({ sourceContentViewID: id - 100, targetContentViewID: id, targetLastModifiedDate: date }); + +describe("advanceContainerDates", () => { + it("advances a container that was in sync at the start and was bumped during the run", () => { + const records = [record(101, T0)]; + + const updates = advanceContainerDates(records, new Map([[101, T0]]), new Map([[101, T1]])); + + expect(updates).toEqual([{ targetContentViewID: 101, from: T0, to: T1 }]); + expect(records[0].targetLastModifiedDate).toBe(T1); + }); + + it("keeps the conflict for a container already changed on the target before the run", () => { + // The pull already had a date after the mapping: a real target edit. + const records = [record(101, T0)]; + + const updates = advanceContainerDates(records, new Map([[101, T1]]), new Map([[101, "10/30/2026 10:10AM"]])); + + expect(updates).toEqual([]); + expect(records[0].targetLastModifiedDate).toBe(T0); + }); + + it("leaves a container whose date didn't move", () => { + const records = [record(101, T0)]; + + expect(advanceContainerDates(records, new Map([[101, T0]]), new Map([[101, T0]]))).toEqual([]); + }); + + it("counts a pulled date older than the mapping as in sync", () => { + const records = [record(101, T0)]; + + expect(advanceContainerDates(records, new Map([[101, TX]]), new Map([[101, T1]]))).toHaveLength(1); + }); + + it("treats a container missing from the pull (created this run) as in sync", () => { + const records = [record(101, T0)]; + + expect(advanceContainerDates(records, new Map(), new Map([[101, T1]]))).toHaveLength(1); + }); + + it("skips records it can't read and containers no longer on the target", () => { + const records = [record(101, "not a date"), record(102, T0), { targetContentViewID: 0, targetLastModifiedDate: T0 }]; + + expect(advanceContainerDates(records, new Map(), new Map([[101, T1]]))).toEqual([]); + }); +}); + +describe("refreshContainerDatesAfterPageSaves", () => { + const SRC = "src-1"; + const TGT = "tgt-1"; + let root: string; + const mappingFile = () => path.join(root, "mappings", `${SRC}-${TGT}`, "containers", "mappings.json"); + + const writeJson = (rel: string, data: any) => { + const file = path.join(root, rel); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); + }; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), "container-dates-")); + // 130: a page's component container, in sync. 131: edited on the target before this run. + writeJson(`${TGT}/containers/130.json`, { contentViewID: 130, referenceName: "home_heading", lastModifiedDate: T0 }); + writeJson(`${TGT}/containers/131.json`, { contentViewID: 131, referenceName: "posts", lastModifiedDate: T1 }); + writeJson(`mappings/${SRC}-${TGT}/containers/mappings.json`, [record(130, T0), record(131, T0)]); + + resetState(); + setState({ rootPath: root, sourceGuid: SRC, targetGuid: TGT, locales: "en-us", isSync: true }); + state.cachedApiClient = { + containerMethods: { + getContainerList: jest.fn(async () => [ + { contentViewID: 130, lastModifiedDate: T1 }, + { contentViewID: 131, lastModifiedDate: "10/30/2026 10:10AM" }, + ]), + }, + } as any; + }); + + afterEach(() => fs.rmSync(root, { recursive: true, force: true })); + + it("advances only the container the page save bumped, and saves the mapping", async () => { + const count = await refreshContainerDatesAfterPageSaves(SRC, TGT, "en-us"); + + expect(count).toBe(1); + const saved = JSON.parse(fs.readFileSync(mappingFile(), "utf8")); + expect(saved.find((r: any) => r.targetContentViewID === 130).targetLastModifiedDate).toBe(T1); + expect(saved.find((r: any) => r.targetContentViewID === 131).targetLastModifiedDate).toBe(T0); + }); + + it("does nothing in preflight", async () => { + state.preflight = true; + + expect(await refreshContainerDatesAfterPageSaves(SRC, TGT, "en-us")).toBe(0); + expect((state.cachedApiClient as any).containerMethods.getContainerList).not.toHaveBeenCalled(); + }); + + it("is bookkeeping only: an API error doesn't throw", async () => { + (state.cachedApiClient as any).containerMethods.getContainerList.mockRejectedValueOnce(new Error("busy")); + + await expect(refreshContainerDatesAfterPageSaves(SRC, TGT, "en-us")).resolves.toBe(0); + }); +}); diff --git a/src/lib/preflight/preflight-report.ts b/src/lib/preflight/preflight-report.ts index 00dba94..3a3e3a7 100644 --- a/src/lib/preflight/preflight-report.ts +++ b/src/lib/preflight/preflight-report.ts @@ -18,7 +18,7 @@ import { state } from "../../core/state"; * the target model (e.g. a missing required field), so a real run skips it and reports a failure. * It is a warning in the plan: not a write, and — unlike "conflict" — it doesn't set the exit code. */ -export type PreflightAction = "create" | "update" | "skip" | "conflict" | "invalid"; +export type PreflightAction = "create" | "update" | "delete" | "skip" | "conflict" | "invalid"; export interface PreflightEntry { /** Phase / element type, e.g. "Models", "Containers", "Content". */ @@ -37,17 +37,19 @@ export interface PreflightPhaseSummary { phase: string; create: number; update: number; + delete: number; skip: number; conflict: number; invalid: number; entries: PreflightEntry[]; } -const ACTION_ORDER: PreflightAction[] = ["create", "update", "skip", "conflict", "invalid"]; +const ACTION_ORDER: PreflightAction[] = ["create", "update", "delete", "skip", "conflict", "invalid"]; const ACTION_COLOR: Record string> = { create: ansiColors.green, update: ansiColors.cyan, + delete: ansiColors.magenta, skip: ansiColors.gray, conflict: ansiColors.red, invalid: ansiColors.yellow, @@ -56,6 +58,7 @@ const ACTION_COLOR: Record string> = { const ACTION_GLYPH: Record = { create: "+", update: "~", + delete: "-", skip: "·", conflict: "✗", invalid: "!", @@ -112,7 +115,7 @@ class PreflightReport { for (const entry of this.entries) { let summary = byPhase.get(entry.phase); if (!summary) { - summary = { phase: entry.phase, create: 0, update: 0, skip: 0, conflict: 0, invalid: 0, entries: [] }; + summary = { phase: entry.phase, create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0, entries: [] }; byPhase.set(entry.phase, summary); } summary[entry.action]++; @@ -123,7 +126,7 @@ class PreflightReport { /** Aggregate totals across all phases. */ getTotals(): Record { - const totals: Record = { create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }; + const totals: Record = { create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }; for (const entry of this.entries) { totals[entry.action]++; } @@ -148,6 +151,7 @@ class PreflightReport { phase: string; create: number; update: number; + delete: number; skip: number; conflict: number; invalid: number; @@ -157,10 +161,11 @@ class PreflightReport { return { totals: this.getTotals(), hasConflicts: this.hasConflicts(), - phases: this.getPhaseSummaries().map(({ phase, create, update, skip, conflict, invalid }) => ({ + phases: this.getPhaseSummaries().map(({ phase, create, update, delete: deletes, skip, conflict, invalid }) => ({ phase, create, update, + delete: deletes, skip, conflict, invalid, @@ -189,7 +194,7 @@ class PreflightReport { const phases = this.getPhaseSummaries(); if (phases.length === 0) { - lines.push(ansiColors.gray("\nNothing to do — no creates, updates, skips, or conflicts detected.\n")); + lines.push(ansiColors.gray("\nNothing to do — no creates, updates, deletes, skips, or conflicts detected.\n")); lines.push(...this.renderWarnings()); return lines.join("\n"); } @@ -199,6 +204,7 @@ class PreflightReport { `\n${ansiColors.bold(phase.phase)}: ` + ACTION_COLOR.create(`${phase.create} create, `) + ACTION_COLOR.update(`${phase.update} update, `) + + (phase.delete > 0 ? ACTION_COLOR.delete(`${phase.delete} delete, `) : "") + ACTION_COLOR.skip(`${phase.skip} skip, `) + ACTION_COLOR.conflict(`${phase.conflict} conflict`) + (phase.invalid > 0 ? ACTION_COLOR.invalid(`, ${phase.invalid} invalid`) : ""); @@ -222,6 +228,7 @@ class PreflightReport { ansiColors.bold("TOTAL: ") + ACTION_COLOR.create(`${totals.create} create, `) + ACTION_COLOR.update(`${totals.update} update, `) + + (totals.delete > 0 ? ACTION_COLOR.delete(`${totals.delete} delete, `) : "") + ACTION_COLOR.skip(`${totals.skip} skip, `) + ACTION_COLOR.conflict(`${totals.conflict} conflict`) + (totals.invalid > 0 ? ACTION_COLOR.invalid(`, ${totals.invalid} invalid`) : "") diff --git a/src/lib/preflight/tests/preflight-report.test.ts b/src/lib/preflight/tests/preflight-report.test.ts index ab10e6b..d702440 100644 --- a/src/lib/preflight/tests/preflight-report.test.ts +++ b/src/lib/preflight/tests/preflight-report.test.ts @@ -121,7 +121,7 @@ describe("hasConflicts", () => { describe("getTotals", () => { it("returns all-zeros totals when there are no entries", () => { setState({ preflight: true }); - expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); it("correctly counts each action type", () => { @@ -133,14 +133,14 @@ describe("getTotals", () => { preflightReport.record({ phase: "Content", action: "skip", name: "c3" }); preflightReport.record({ phase: "Pages", action: "conflict", name: "p1" }); - expect(preflightReport.getTotals()).toEqual({ create: 2, update: 1, skip: 2, conflict: 1, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 2, update: 1, delete: 0, skip: 2, conflict: 1, invalid: 0 }); }); it("counts only the actions that appear", () => { setState({ preflight: true }); preflightReport.record({ phase: "Models", action: "update", name: "m1" }); - expect(preflightReport.getTotals()).toEqual({ create: 0, update: 1, skip: 0, conflict: 0, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 1, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); }); @@ -292,7 +292,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); - expect(json.totals).toEqual({ create: 1, update: 1, skip: 1, conflict: 1, invalid: 0 }); + expect(json.totals).toEqual({ create: 1, update: 1, delete: 0, skip: 1, conflict: 1, invalid: 0 }); expect(json.hasConflicts).toBe(true); }); @@ -306,7 +306,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); expect(json.phases.map((p) => p.phase)).toEqual(["Models", "Containers"]); - expect(json.phases[0]).toEqual({ phase: "Models", create: 2, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(json.phases[0]).toEqual({ phase: "Models", create: 2, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); it("carries every entry through, with locale and detail intact", () => { @@ -326,7 +326,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); - expect(json.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(json.totals).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); expect(json.hasConflicts).toBe(false); expect(json.phases).toEqual([]); expect(json.entries).toEqual([]); @@ -392,10 +392,10 @@ describe("invalid action (PROD-2542)", () => { detail: "would create but Missing required field(s) on model PromoBanner: promoTitle (Text)", }); - expect(preflightReport.getTotals()).toEqual({ create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }); + expect(preflightReport.getTotals()).toEqual({ create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }); expect(preflightReport.hasConflicts()).toBe(false); expect(preflightReport.toJSON().phases).toEqual([ - { phase: "Content", create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }, + { phase: "Content", create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }, ]); }); @@ -459,3 +459,23 @@ describe("recordWarning", () => { expect(preflightReport.getWarnings()).toEqual([]); }); }); + +// ─── delete action (--syncDeletes) ──────────────────────────────────────────── + +describe("delete action (--syncDeletes)", () => { + beforeEach(() => { + setState({ preflight: true }); + preflightReport.reset(); + }); + + it("is counted on its own and rendered in the phase header and totals", () => { + preflightReport.record({ phase: "Delete Content", action: "delete", name: "posts #12", locale: "en-us", detail: "deleted in the source" }); + preflightReport.record({ phase: "Delete Content", action: "conflict", name: "posts #13", locale: "en-us" }); + + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, delete: 1, skip: 0, conflict: 1, invalid: 0 }); + // eslint-disable-next-line no-control-regex + const table = preflightReport.renderTable().replace(/\x1b\[[0-9;]*m/g, ""); + expect(table).toContain("Delete Content: 0 create, 0 update, 1 delete, 0 skip, 1 conflict"); + expect(table).toContain("- [en-us] posts #12 — deleted in the source"); + }); +}); diff --git a/src/lib/pushers/asset-pusher.ts b/src/lib/pushers/asset-pusher.ts index 0c84ec4..ea034af 100644 --- a/src/lib/pushers/asset-pusher.ts +++ b/src/lib/pushers/asset-pusher.ts @@ -262,7 +262,9 @@ export async function pushAssets( const hasTargetChanges = existingMapping !== null && referenceMapper.hasTargetChanged(targetAsset); const hasSourceChanges = existingMapping !== null && referenceMapper.hasSourceChanged(media); - const isConflict = existingMapping !== null && hasTargetChanges && hasSourceChanges; + // The target changed since the last sync: a conflict whether or not the source changed too, so + // a target-only edit is kept (and reported) unless --overwrite says to replace it. + const isConflict = existingMapping !== null && hasTargetChanges; const isOverwrite = state.overwrite; const shouldSkip = existingMapping !== null && !hasTargetChanges && !hasSourceChanges; // Update if: @@ -322,14 +324,19 @@ export async function pushAssets( preflightReport.record({ phase: "Assets", action: "skip", name: media.fileName, detail: "up to date" }); skipped++; } else if (isConflict) { - // Conflict: both source and target changed and --overwrite was not set. - // The real sync silently does nothing here; surface it in preflight. + // Conflict: the target changed since the last sync and --overwrite was not set. + logger.asset.skipped( + media, + "conflict: target changed since the last sync; use --overwrite to replace it", + targetGuid + ); preflightReport.record({ phase: "Assets", action: "conflict", name: media.fileName, detail: "target changed; use --overwrite to force", }); + skipped++; } } catch (error: any) { const errorMsg = extractErrorMessage(error); diff --git a/src/lib/pushers/content-pusher/content-batch-processor.ts b/src/lib/pushers/content-pusher/content-batch-processor.ts index 58047e2..6fd2efe 100644 --- a/src/lib/pushers/content-pusher/content-batch-processor.ts +++ b/src/lib/pushers/content-pusher/content-batch-processor.ts @@ -495,7 +495,13 @@ export class ContentBatchProcessor { const isTargetContentItemDeleted = existingMapping && existingTargetContentItem == null; let existingContentID = -1; - if (isTargetContentItemDeleted && state.overwrite) { + // PROD-3076: pass 2 of a reference cycle updates an item pass 1 created moments ago; it is + // not in the pulled target data yet, so without this it would look deleted and be recreated. + const createdThisRun = !!this.config.trustSameRunMappings && !!existingMapping && !existingTargetContentItem; + + if (createdThisRun) { + existingContentID = existingMapping.targetContentID; + } else if (isTargetContentItemDeleted && state.overwrite) { existingContentID = existingMapping ? existingMapping?.targetContentID : -1; } else { existingContentID = existingTargetContentItem ? existingTargetContentItem.contentID : -1; @@ -585,7 +591,9 @@ export class ContentBatchProcessor { // Only an update of a target item known to exist (or a locale version of one found in // another locale) must come back with the same ID. expectedId: - existingTargetContentItem || (!existingMapping && existingContentID > 0) ? existingContentID : undefined, + existingTargetContentItem || createdThisRun || (!existingMapping && existingContentID > 0) + ? existingContentID + : undefined, }); } catch (error: any) { const prepareError = error.message || "payload preparation failed"; diff --git a/src/lib/pushers/content-pusher/content-pusher.ts b/src/lib/pushers/content-pusher/content-pusher.ts index 1c2c076..83b7884 100644 --- a/src/lib/pushers/content-pusher/content-pusher.ts +++ b/src/lib/pushers/content-pusher/content-pusher.ts @@ -10,6 +10,8 @@ import { ContentItem, Model } from "@agility/management-sdk"; import { ContainerMapper } from "lib/mappers/container-mapper"; import { getApiClient } from "core/state"; import { reportForeignCdnReferences } from "./util/report-foreign-cdn-references"; +import { BatchProcessingResult } from "./util/types"; +import { stripContentReferences } from "./util/strip-content-references"; /** * Push content to the target instance @@ -33,13 +35,15 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content return { status: "success" as const, successful: 0, failed: 0, skipped: 0, publishableIds: [], failureDetails: [] }; } - // Deterministically classify content items based on list references (fulllist=true) - const { normalContentItems, linkedContentItems, skippedItems } = getContentItemTypes(contentItems, { - containerMapper, - modelMapper, - referenceMapper, - logger: logger as any, - }); + // Deterministically classify content items by references: linked (referenced, split into + // dependency levels), normal (unreferenced) and skipped (missing mappings) + const { normalContentItems, linkedContentItems, linkedContentLevels, linkedCycleReferences, skippedItems } = + getContentItemTypes(contentItems, { + containerMapper, + modelMapper, + referenceMapper, + logger: logger as any, + }); // PROD-2773: warn about asset URLs into a CDN container owned by neither instance. Reporting // only — runs before change detection (and in --preflight) and never alters an item. Items @@ -99,6 +103,37 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content } }; + // Fold one processBatches result into the totals; register each failure so the next pass's + // parents and the page pusher can say why their reference is unresolved. Items in `heldBack` + // were saved as a pass-1 stub (PROD-3076): pass 2 counts them, and only pass 2's version is + // published, so their successes here are left out. + const recordBatchResult = (result: BatchProcessingResult, heldBack: Set = new Set()): void => { + const heldBackTargetIds = new Set( + (result.successfulItems || []) + .filter((item) => heldBack.has(item.originalContent?.contentID)) + .map((item) => item.newContentId) + ); + totalSuccessful += result.successCount - heldBackTargetIds.size; + totalFailed += result.failureCount; + totalSkipped += result.skippedCount; + allPublishableIds.push(...result.publishableIds.filter((id) => !heldBackTargetIds.has(id))); + for (const item of result.failedItems || []) { + const name = item.originalContent?.properties?.referenceName || "Unknown"; + const contentID = item.originalContent?.contentID; + allFailureDetails.push({ + name, + error: item.error, + type: "content", + contentID, + guid: sourceGuidStr, + locale, + }); + if (contentID) { + registerFailedContent(contentID, name, item.error, locale); + } + } + }; + try { // Import getApiClient for both batch configurations @@ -107,7 +142,8 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content totalSkipped += skippedItems.length; } - // Process linked content items second (with dependencies) + // Process linked content items first, one dependency level at a time (leaves first), so + // the mappings each level references were written by an earlier level (PROD-3025). if (linkedContentItems.length > 0) { const linkedBatchConfig = { apiClient: getApiClient(), @@ -139,37 +175,31 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content totalSuccessful += filteredLinkedContentItems.itemsToProcess.length; totalSkipped += filteredLinkedContentItems.skippedCount; } else { - const linkedBatchProcessor = new ContentBatchProcessor(linkedBatchConfig); - const linkedResult = await linkedBatchProcessor.processBatches( - filteredLinkedContentItems.itemsToProcess.reverse(), - logger, - "Linked Content" - ); + totalSkipped += filteredLinkedContentItems.skippedCount; + const levels = splitIntoLevels(filteredLinkedContentItems.itemsToProcess, linkedContentLevels); + const linkedBatchProcessor = new ContentBatchProcessor(linkedBatchConfig); + const cycleBatchProcessor = new ContentBatchProcessor({ ...linkedBatchConfig, trustSameRunMappings: true }); + for (let i = 0; i < levels.length; i++) { + const label = levels.length > 1 ? `Linked Content (level ${i + 1}/${levels.length})` : "Linked Content"; + + // PROD-3076: items in a reference cycle go out first with their references to not-yet- + // created cycle members blanked, then again in full once every member has a target ID. + const { toPush, stubbed } = stubCycleReferences(levels[i], linkedCycleReferences, referenceMapper, modelMapper); + const result = await linkedBatchProcessor.processBatches(toPush, logger, label); + recordBatchResult(result, new Set(stubbed.map((item) => item.contentID))); - totalSuccessful += linkedResult.successCount; - totalFailed += linkedResult.failureCount; - totalSkipped += filteredLinkedContentItems.skippedCount; - totalSkipped += linkedResult.skippedCount; - allPublishableIds.push(...linkedResult.publishableIds); - // Collect failure details for error summary and register in failed content registry - if (linkedResult.failedItems && linkedResult.failedItems.length > 0) { - linkedResult.failedItems.forEach((item) => { - const name = item.originalContent?.properties?.referenceName || "Unknown"; - const contentID = item.originalContent?.contentID; - allFailureDetails.push({ - name, - error: item.error, - type: "content", - contentID, - guid: sourceGuidStr, - locale, - }); - // Register in global registry so page pusher can provide better error messages - if (contentID) { - registerFailedContent(contentID, name, item.error, locale); + const created = new Set((result.successfulItems || []).map((item) => item.originalContent?.contentID)); + const toComplete = stubbed.filter((item) => created.has(item.contentID)); + if (toComplete.length > 0) { + const completed = await cycleBatchProcessor.processBatches(toComplete, logger, `${label}: completing reference cycles`); + recordBatchResult(completed); + // Saved without its references: make the next sync push it again + for (const failed of completed.failedItems || []) { + const contentID = failed.originalContent?.contentID; + if (contentID) referenceMapper.markSourceChanged(contentID); + } } - }); - } + } } } @@ -202,37 +232,15 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content totalSuccessful += filteredNormalContentItems.itemsToProcess.length; totalSkipped += filteredNormalContentItems.skippedCount; } else { - const normalBatchProcessor = new ContentBatchProcessor(normalBatchConfig); - const normalResult = await normalBatchProcessor.processBatches( - filteredNormalContentItems.itemsToProcess as ContentItem[], - logger, - "Normal Content" - ); + const normalBatchProcessor = new ContentBatchProcessor(normalBatchConfig); + const normalResult = await normalBatchProcessor.processBatches( + filteredNormalContentItems.itemsToProcess as ContentItem[], + logger, + "Normal Content" + ); - totalSuccessful += normalResult.successCount; - totalFailed += normalResult.failureCount; - totalSkipped += filteredNormalContentItems.skippedCount; - totalSkipped += normalResult.skippedCount; - allPublishableIds.push(...normalResult.publishableIds); - // Collect failure details for error summary and register in failed content registry - if (normalResult.failedItems && normalResult.failedItems.length > 0) { - normalResult.failedItems.forEach((item) => { - const name = item.originalContent?.properties?.referenceName || "Unknown"; - const contentID = item.originalContent?.contentID; - allFailureDetails.push({ - name, - error: item.error, - type: "content", - contentID, - guid: sourceGuidStr, - locale, - }); - // Register in global registry so page pusher can provide better error messages - if (contentID) { - registerFailedContent(contentID, name, item.error, locale); - } - }); - } + totalSkipped += filteredNormalContentItems.skippedCount; + recordBatchResult(normalResult); } } @@ -257,3 +265,62 @@ export async function pushContent(sourceData: ContentItem[], targetData: Content }; } } + +/** + * Groups the items left after filtering by their dependency level (PROD-3025), dropping levels + * the filter emptied. An item missing from every level goes last, after everything it could need. + */ +export function splitIntoLevels(items: ContentItem[], levels: ContentItem[][]): ContentItem[][] { + const levelById = new Map(); + levels.forEach((level, i) => level.forEach((item) => levelById.set(item.contentID, i))); + + const grouped: ContentItem[][] = levels.map(() => []); + const unplaced: ContentItem[] = []; + for (const item of items) { + const level = levelById.get(item.contentID); + if (level === undefined) unplaced.push(item); + else grouped[level].push(item); + } + grouped.push(unplaced); + return grouped.filter((level) => level.length > 0); +} + +/** + * PROD-3076: for the items of one level, the copies to push in pass 1. An item in a reference cycle + * whose cycle partners (or itself) have no target ID yet goes out with those references blanked, + * and is returned in `stubbed` for pass 2. Everything else is pushed as-is. Partners that already + * have a mapping (a re-sync) are left alone, so a re-sync never blanks anything. + */ +export function stubCycleReferences( + items: ContentItem[], + cycleReferences: Map, + referenceMapper: ContentItemMapper, + modelMapper: ModelMapper +): { toPush: ContentItem[]; stubbed: ContentItem[] } { + const toPush: ContentItem[] = []; + const stubbed: ContentItem[] = []; + + for (const item of items) { + const unmapped = (cycleReferences.get(item.contentID) || []).filter( + (id) => !referenceMapper.getContentItemMappingByContentID(id, "source") + ); + if (unmapped.length === 0) { + toPush.push(item); + continue; + } + + const definitionName = item.properties?.definitionName?.toLowerCase(); + const modelMapping = definitionName ? modelMapper.getModelMappingByReferenceName(definitionName, "source") : null; + const sourceModel = modelMapping ? modelMapper.getMappedEntity(modelMapping, "source") : null; + const { fields, stripped } = stripContentReferences(item.fields, new Set(unmapped), sourceModel); + if (stripped === 0) { + toPush.push(item); + continue; + } + + toPush.push({ ...item, fields }); + stubbed.push(item); + } + + return { toPush, stubbed }; +} diff --git a/src/lib/pushers/content-pusher/tests/content-batch-processor.test.ts b/src/lib/pushers/content-pusher/tests/content-batch-processor.test.ts index 38b26e0..b53eefc 100644 --- a/src/lib/pushers/content-pusher/tests/content-batch-processor.test.ts +++ b/src/lib/pushers/content-pusher/tests/content-batch-processor.test.ts @@ -625,6 +625,60 @@ describe("ContentBatchProcessor.prepareContentPayloads — stale container mappi }); }); +// ─── PROD-3076: updating an item created earlier in the same run ───────────── + +describe("ContentBatchProcessor.prepareContentPayloads — trustSameRunMappings (PROD-3076)", () => { + // The content item is mapped (source 1 → target 77) but target 77 is not in the pulled target data, + // which is the state pass 2 of a reference cycle sees: pass 1 created it moments ago. + function seed(sourceGuid: string, targetGuid: string): ContentItemMapper { + writeEntityFile(tmpDir, sourceGuid, "models", 1, { id: 1, referenceName: "TestModel", fields: [] }); + writeMappingFile(tmpDir, sourceGuid, targetGuid, "models", [ + { sourceGuid, targetGuid, sourceID: 1, targetID: 1, sourceReferenceName: "TestModel", targetReferenceName: "TestModel", + sourceLastModifiedDate: "07/01/2026 12:00PM", targetLastModifiedDate: "07/01/2026 12:00PM" }, + ]); + writeMappingFile(tmpDir, sourceGuid, targetGuid, "containers", [ + { sourceGuid, targetGuid, sourceContentViewID: 10, targetContentViewID: 999, sourceReferenceName: "ref-1", + targetReferenceName: "ref-1", sourceLastModifiedDate: "07/01/2026 12:00PM", targetLastModifiedDate: "07/01/2026 12:00PM" }, + ]); + writeEntityFile(tmpDir, targetGuid, "containers", 999, { contentViewID: 999, referenceName: "ref-1" }); + const referenceMapper = new ContentItemMapper(sourceGuid, targetGuid, "en-us"); + referenceMapper.addMapping( + { contentID: 1, properties: { versionID: 1 } } as any, + { contentID: 77, properties: { versionID: 5 } } as any + ); + return referenceMapper; + } + + function item(): any { + const it = makeContentItem(1); + it.properties.referenceName = "ref-1"; + it.properties.definitionName = "TestModel"; + return it; + } + + it("updates the mapped target item instead of creating another one", async () => { + const referenceMapper = seed("trust-src", "trust-tgt"); + const processor = new ContentBatchProcessor( + makeConfig({ sourceGuid: "trust-src", targetGuid: "trust-tgt", referenceMapper, trustSameRunMappings: true }) + ); + + const result = await (processor as any).prepareContentPayloads([item()], "trust-src", "trust-tgt", makeLogger()); + + expect(result.prepared).toHaveLength(1); + expect(result.prepared[0].payload.contentID).toBe(77); + expect(result.prepared[0].expectedId).toBe(77); + }); + + it("without the option, treats the missing target as gone (unchanged behavior)", async () => { + const referenceMapper = seed("notrust-src", "notrust-tgt"); + const processor = new ContentBatchProcessor(makeConfig({ sourceGuid: "notrust-src", targetGuid: "notrust-tgt", referenceMapper })); + + const result = await (processor as any).prepareContentPayloads([item()], "notrust-src", "notrust-tgt", makeLogger()); + + expect(result.prepared[0].payload.contentID).toBe(-1); + }); +}); + // ─── PROD-2542: payload/source alignment ────────────────────────────────────── // // These use the REAL extractContentBatchResults (the module is mocked above for the other tests), so diff --git a/src/lib/pushers/content-pusher/tests/content-pusher-linked-levels.test.ts b/src/lib/pushers/content-pusher/tests/content-pusher-linked-levels.test.ts new file mode 100644 index 0000000..a3c6b44 --- /dev/null +++ b/src/lib/pushers/content-pusher/tests/content-pusher-linked-levels.test.ts @@ -0,0 +1,348 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { resetState, setState, state, getFailedContent } from "core/state"; + +// PROD-3025: pushContent pushes the linked tier one dependency level at a time, leaves first. +// Classification, filtering and batching are mocked so only content-pusher's own ordering runs. + +jest.mock("../util/get-content-item-types", () => ({ + getContentItemTypes: jest.fn(), +})); + +jest.mock("../util/filter-content-items-for-processing", () => ({ + filterContentItemsForProcessing: jest.fn(), +})); + +const mockProcessBatches = jest.fn(); +jest.mock("../content-batch-processor", () => ({ + ContentBatchProcessor: jest.fn().mockImplementation(() => ({ processBatches: mockProcessBatches })), +})); + +import { getContentItemTypes } from "../util/get-content-item-types"; +import { filterContentItemsForProcessing } from "../util/filter-content-items-for-processing"; +import { pushContent, splitIntoLevels, stubCycleReferences } from "../content-pusher"; +import { ContentBatchProcessor } from "../content-batch-processor"; +import { ContentItemMapper } from "lib/mappers/content-item-mapper"; + +const mockTypes = getContentItemTypes as jest.Mock; +const mockFilter = filterContentItemsForProcessing as jest.Mock; + +let tmpDir: string; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agility-cpl-")); +}); + +afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + resetState(); + state.failedContentRegistry.clear(); + setState({ rootPath: tmpDir, sourceGuid: "src-guid", targetGuid: "tgt-guid" }); + const logger = { + content: { error: jest.fn(), created: jest.fn(), skipped: jest.fn() }, + warning: jest.fn(), + getGuid: () => "src-guid", + setGuid: jest.fn(), + }; + state.loggerRegistry = new Map([["src-guid", logger as any]]); + state.cachedApiClient = {} as any; + mockProcessBatches.mockReset(); + mockFilter.mockReset(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => { + jest.restoreAllMocks(); +}); + +function item(id: number): any { + return { contentID: id, properties: { referenceName: `ref-${id}`, definitionName: "Model", state: 2 }, fields: {} }; +} + +function filterResult(itemsToProcess: any[], skippedCount = 0): any { + return { itemsToProcess, itemsToSkip: [], skippedCount, itemsToFail: [] }; +} + +function batchResult(items: any[], failed: any[] = []): any { + return { + successCount: items.length, + failureCount: failed.length, + skippedCount: 0, + successfulItems: [], + failedItems: failed.map((f) => ({ originalContent: f, error: "boom" })), + publishableIds: items.map((i) => i.contentID + 1000), + }; +} + +describe("pushContent — linked dependency levels (PROD-3025)", () => { + it("pushes each level in its own processBatches call, leaves first, then the normal tier", async () => { + const category = item(1); + const post = item(2); + const featured = item(3); + mockTypes.mockReturnValue({ + linkedContentItems: [post, category], + linkedContentLevels: [[category], [post]], linkedCycleReferences: new Map(), + normalContentItems: [featured], + skippedItems: [], + }); + // The filter returns the linked tier in arbitrary order; the levels decide the push order + mockFilter.mockResolvedValueOnce(filterResult([post, category])).mockResolvedValueOnce(filterResult([featured])); + mockProcessBatches.mockImplementation(async (items: any[]) => batchResult(items)); + + const result = await pushContent([featured, post, category], [], "en-us"); + + const calls = mockProcessBatches.mock.calls.map((c) => [c[0].map((i: any) => i.contentID), c[2]]); + expect(calls).toEqual([ + [[1], "Linked Content (level 1/2)"], + [[2], "Linked Content (level 2/2)"], + [[3], "Normal Content"], + ]); + expect(result.status).toBe("success"); + expect(result.successful).toBe(3); + expect(result.publishableIds).toEqual([1001, 1002, 1003]); + }); + + it("adds up failures across levels and registers each one", async () => { + const leaf = item(1); + const mid = item(2); + mockTypes.mockReturnValue({ + linkedContentItems: [leaf, mid], + linkedContentLevels: [[leaf], [mid]], linkedCycleReferences: new Map(), + normalContentItems: [], + skippedItems: [], + }); + mockFilter.mockResolvedValueOnce(filterResult([leaf, mid], 2)); + mockProcessBatches + .mockResolvedValueOnce(batchResult([leaf])) + .mockResolvedValueOnce(batchResult([], [mid])); + + const result = await pushContent([leaf, mid], [], "en-us"); + + expect(result.status).toBe("error"); + expect(result.successful).toBe(1); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(2); + expect(result.failureDetails).toEqual([expect.objectContaining({ contentID: 2, error: "boom" })]); + expect(getFailedContent(2, "en-us")).toBeDefined(); + }); + + it("does not call the batch processor for a level the filter emptied", async () => { + const leaf = item(1); + const mid = item(2); + mockTypes.mockReturnValue({ + linkedContentItems: [leaf, mid], + linkedContentLevels: [[leaf], [mid]], linkedCycleReferences: new Map(), + normalContentItems: [], + skippedItems: [], + }); + // leaf is unchanged on the target, so only mid is left to push + mockFilter.mockResolvedValueOnce(filterResult([mid], 1)); + mockProcessBatches.mockImplementation(async (items: any[]) => batchResult(items)); + + await pushContent([leaf, mid], [], "en-us"); + + expect(mockProcessBatches).toHaveBeenCalledTimes(1); + expect(mockProcessBatches.mock.calls[0][0]).toEqual([mid]); + expect(mockProcessBatches.mock.calls[0][2]).toBe("Linked Content"); + }); + + it("writes nothing in preflight", async () => { + setState({ preflight: true }); + const leaf = item(1); + const mid = item(2); + mockTypes.mockReturnValue({ + linkedContentItems: [leaf, mid], + linkedContentLevels: [[leaf], [mid]], linkedCycleReferences: new Map(), + normalContentItems: [], + skippedItems: [], + }); + mockFilter.mockResolvedValueOnce(filterResult([leaf, mid])); + + const result = await pushContent([leaf, mid], [], "en-us"); + + expect(mockProcessBatches).not.toHaveBeenCalled(); + expect(result.successful).toBe(2); + }); +}); + +describe("splitIntoLevels", () => { + it("groups items by level and drops empty levels", () => { + const a = item(1); + const b = item(2); + const c = item(3); + expect(splitIntoLevels([c, a], [[a], [b], [c]])).toEqual([[a], [c]]); + }); + + it("puts an item missing from every level last", () => { + const a = item(1); + const stray = item(9); + expect(splitIntoLevels([stray, a], [[a]])).toEqual([[a], [stray]]); + }); +}); + +// ─── reference cycles: two passes in the same run (PROD-3076) ──────────────── + +describe("pushContent — reference cycles (PROD-3076)", () => { + // A and B reference each other through a single-item linked field + function cycle(): { a: any; b: any } { + const a = item(1); + const b = item(2); + a.fields = { next: { contentid: 2, fulllist: false } }; + b.fields = { next: { contentid: 1, fulllist: false } }; + return { a, b }; + } + + function classify(a: any, b: any): void { + mockTypes.mockReturnValue({ + linkedContentItems: [a, b], + linkedContentLevels: [[a, b]], + linkedCycleReferences: new Map([ + [1, [2]], + [2, [1]], + ]), + normalContentItems: [], + skippedItems: [], + }); + mockFilter.mockResolvedValueOnce(filterResult([a, b])); + } + + // Every item sent succeeds; target ID = source ID + 1000 + function succeedAll(failIds: number[] = []): void { + mockProcessBatches.mockImplementation(async (items: any[]) => { + const ok = items.filter((i) => !failIds.includes(i.contentID)); + const bad = items.filter((i) => failIds.includes(i.contentID)); + return { + successCount: ok.length, + failureCount: bad.length, + skippedCount: 0, + successfulItems: ok.map((i) => ({ originalContent: i, newItem: {}, newContentId: i.contentID + 1000 })), + failedItems: bad.map((i) => ({ originalContent: i, error: "boom" })), + publishableIds: ok.map((i) => i.contentID + 1000), + }; + }); + } + + it("creates the members with the cycle references blanked, then sends them in full", async () => { + const { a, b } = cycle(); + classify(a, b); + succeedAll(); + + await pushContent([a, b], [], "en-us"); + + const [pass1, pass2] = mockProcessBatches.mock.calls; + expect(pass1[0].map((i: any) => i.fields.next.contentid)).toEqual([-1, -1]); + expect(pass1[2]).toBe("Linked Content"); + expect(pass2[0]).toEqual([a, b]); + expect(pass2[2]).toBe("Linked Content: completing reference cycles"); + // Pass 2 runs through a processor that updates the targets pass 1 just created + const configs = (ContentBatchProcessor as unknown as jest.Mock).mock.calls.map((c) => c[0]); + expect(configs.some((c) => c.trustSameRunMappings === true)).toBe(true); + // The source items themselves are never modified + expect(a.fields.next.contentid).toBe(2); + }); + + it("counts and publishes each member once", async () => { + const { a, b } = cycle(); + classify(a, b); + succeedAll(); + + const result = await pushContent([a, b], [], "en-us"); + + expect(result.successful).toBe(2); + expect(result.failed).toBe(0); + expect(result.publishableIds).toEqual([1001, 1002]); + }); + + it("reports a failed second pass and makes the next sync push the item again", async () => { + const { a, b } = cycle(); + classify(a, b); + let call = 0; + mockProcessBatches.mockImplementation(async (items: any[]) => { + call++; + const fail = call === 2 ? [2] : []; + const ok = items.filter((i) => !fail.includes(i.contentID)); + return { + successCount: ok.length, + failureCount: items.length - ok.length, + skippedCount: 0, + successfulItems: ok.map((i) => ({ originalContent: i, newItem: {}, newContentId: i.contentID + 1000 })), + failedItems: items.filter((i) => fail.includes(i.contentID)).map((i) => ({ originalContent: i, error: "boom" })), + publishableIds: ok.map((i) => i.contentID + 1000), + }; + }); + const markSpy = jest.spyOn(ContentItemMapper.prototype, "markSourceChanged"); + + const result = await pushContent([a, b], [], "en-us"); + + expect(result.successful).toBe(1); + expect(result.failed).toBe(1); + // The half-saved stub of B is not published + expect(result.publishableIds).toEqual([1001]); + expect(markSpy).toHaveBeenCalledWith(2); + }); + + it("does not run a second pass for a member whose stub failed", async () => { + const { a, b } = cycle(); + classify(a, b); + succeedAll([2]); + + const result = await pushContent([a, b], [], "en-us"); + + expect(mockProcessBatches.mock.calls[1][0]).toEqual([a]); + expect(result.successful).toBe(1); + expect(result.failed).toBe(1); + }); + + it("sends a cycle whose members are already mapped (a re-sync) once, unchanged", async () => { + const { a, b } = cycle(); + classify(a, b); + succeedAll(); + jest.spyOn(ContentItemMapper.prototype, "getContentItemMappingByContentID").mockReturnValue({} as any); + + await pushContent([a, b], [], "en-us"); + + expect(mockProcessBatches).toHaveBeenCalledTimes(1); + expect(mockProcessBatches.mock.calls[0][0]).toEqual([a, b]); + }); +}); + +describe("stubCycleReferences (PROD-3076)", () => { + const mapper = (mapped: number[]): any => ({ + getContentItemMappingByContentID: (id: number) => (mapped.includes(id) ? {} : null), + }); + const modelMapper: any = { getModelMappingByReferenceName: () => null, getMappedEntity: () => null }; + + it("blanks only the references to unmapped members", () => { + const a = item(1); + a.fields = { next: { contentid: 2 }, other: { contentid: 3 }, picks: { sortids: "2,3" } }; + const { toPush, stubbed } = stubCycleReferences([a], new Map([[1, [2, 3]]]), mapper([3]), modelMapper); + + expect(toPush[0].fields).toEqual({ next: { contentid: -1 }, other: { contentid: 3 }, picks: { sortids: "3" } }); + expect(stubbed).toEqual([a]); + }); + + it("passes items outside a cycle through untouched", () => { + const a = item(1); + a.fields = { next: { contentid: 2 } }; + const { toPush, stubbed } = stubCycleReferences([a], new Map(), mapper([]), modelMapper); + + expect(toPush).toEqual([a]); + expect(toPush[0]).toBe(a); + expect(stubbed).toEqual([]); + }); + + it("stubs a self-reference", () => { + const c = item(3); + c.fields = { next: { contentid: 3 } }; + const { toPush, stubbed } = stubCycleReferences([c], new Map([[3, [3]]]), mapper([]), modelMapper); + + expect(toPush[0].fields.next.contentid).toBe(-1); + expect(stubbed).toEqual([c]); + }); +}); diff --git a/src/lib/pushers/content-pusher/tests/content-pusher-validation.test.ts b/src/lib/pushers/content-pusher/tests/content-pusher-validation.test.ts index f76f814..92ef2a6 100644 --- a/src/lib/pushers/content-pusher/tests/content-pusher-validation.test.ts +++ b/src/lib/pushers/content-pusher/tests/content-pusher-validation.test.ts @@ -99,20 +99,14 @@ describe("pushContent — client-side validation failures (PROD-2542)", () => { const normalGood = item(2); mockTypes.mockReturnValue({ linkedContentItems: [linkedBad], + linkedContentLevels: [[linkedBad]], linkedCycleReferences: new Map(), normalContentItems: [normalBad, normalGood], skippedItems: [], }); mockFilter .mockResolvedValueOnce(filterResult([], [invalid(linkedBad)])) .mockResolvedValueOnce(filterResult([normalGood], [invalid(normalBad)])); - mockProcessBatches.mockResolvedValueOnce({ - successCount: 0, - failureCount: 0, - skippedCount: 0, - successfulItems: [], - failedItems: [], - publishableIds: [], - }); + // The linked tier is empty after filtering, so only the normal pass reaches the batch processor. mockProcessBatches.mockResolvedValueOnce({ successCount: 1, failureCount: 0, @@ -157,7 +151,7 @@ describe("pushContent — client-side validation failures (PROD-2542)", () => { it("registers a failure (locale-scoped) before the next pass runs", async () => { const linkedBad = item(10, "linked-bad"); const parent = item(20, "parent"); - mockTypes.mockReturnValue({ linkedContentItems: [linkedBad], normalContentItems: [parent], skippedItems: [] }); + mockTypes.mockReturnValue({ linkedContentItems: [linkedBad], linkedContentLevels: [[linkedBad]], linkedCycleReferences: new Map(), normalContentItems: [parent], skippedItems: [] }); let seenDuringNormalPass: any; mockFilter.mockResolvedValueOnce(filterResult([], [invalid(linkedBad)])).mockImplementationOnce(async () => { seenDuringNormalPass = getFailedContent(10, "en-us"); @@ -180,7 +174,7 @@ describe("pushContent — client-side validation failures (PROD-2542)", () => { setState({ preflight: true }); const bad = item(12942); const good = item(2); - mockTypes.mockReturnValue({ linkedContentItems: [], normalContentItems: [bad, good], skippedItems: [] }); + mockTypes.mockReturnValue({ linkedContentItems: [], linkedContentLevels: [], linkedCycleReferences: new Map(), normalContentItems: [bad, good], skippedItems: [] }); mockFilter.mockResolvedValueOnce(filterResult([good], [invalid(bad)])); const result = await pushContent([bad, good], [], "en-us"); diff --git a/src/lib/pushers/content-pusher/tests/content-pusher.test.ts b/src/lib/pushers/content-pusher/tests/content-pusher.test.ts index 33ad15b..7d881b9 100644 --- a/src/lib/pushers/content-pusher/tests/content-pusher.test.ts +++ b/src/lib/pushers/content-pusher/tests/content-pusher.test.ts @@ -125,6 +125,7 @@ describe("pushContent — batch processing error handling", () => { getContentItemTypes: jest.fn().mockReturnValue({ normalContentItems: [makeContentItem(1)], linkedContentItems: [], + linkedContentLevels: [], linkedCycleReferences: new Map(), skippedItems: [], }), })); diff --git a/src/lib/pushers/content-pusher/util/change-detection.ts b/src/lib/pushers/content-pusher/util/change-detection.ts index 163b4ed..e150285 100644 --- a/src/lib/pushers/content-pusher/util/change-detection.ts +++ b/src/lib/pushers/content-pusher/util/change-detection.ts @@ -136,6 +136,42 @@ export function changeDetection( } } + // TARGET UPDATE ONLY: edited on the target since the last sync, the source unchanged. A conflict, + // like a change on both sides (and like pages, models and containers): reported and skipped, or + // replaced with the source under --overwrite. It used to fall through to "up to date", so the + // target kept a version the source never had and nothing said so. A mapping with no recorded + // target version (older mapping files) can't tell, so it is left alone. + if ( + sourceVersion > 0 && + targetVersion > 0 && + mappedTargetVersion > 0 && + sourceVersion <= mappedSourceVersion && + targetVersion > mappedTargetVersion + ) { + if (overwrite) { + return { + entity: targetEntity, + shouldUpdate: true, + shouldCreate: false, + shouldSkip: false, + isConflict: false, + reason: "Overwrite mode enabled: replacing the target's own changes with the source", + }; + } + const sourceUrl = `https://app.agilitycms.com/instance/${state.sourceGuid}/${locale}/content/listitem-${sourceEntity.contentID}`; + const targetUrl = `https://app.agilitycms.com/instance/${state.targetGuid}/${locale}/content/listitem-${targetEntity.contentID}`; + return { + entity: targetEntity, + shouldUpdate: false, + shouldCreate: false, + shouldSkip: false, + isConflict: true, + reason: + `The target changed since the last sync and the source didn't. Use --overwrite to replace it with the source.\n` + + ` - source: ${sourceUrl} \n - target: ${targetUrl}`, + }; + } + if (sourceVersion > mappedSourceVersion && targetVersion <= mappedTargetVersion) { //SOURCE UPDATE ONLY // Source version is newer the mapped source version diff --git a/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts b/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts index 21a2b10..563883a 100644 --- a/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts +++ b/src/lib/pushers/content-pusher/util/filter-content-items-for-processing.ts @@ -129,7 +129,9 @@ export async function filterContentItemsForProcessing({ // pick the headline from the reason rather than hard-coding "changes detected in both". const headline = /no longer exists/i.test(reason || "") ? "mapped target item no longer exists" - : "changes detected in both source and target"; + : /source didn't/i.test(reason || "") + ? "the target changed since the last sync" + : "changes detected in both source and target"; console.warn( `⚠️ Conflict detected content ${ansiColors.underline(itemName)} ${ansiColors.bold.grey(headline)}. Please resolve manually.` ); diff --git a/src/lib/pushers/content-pusher/util/get-content-item-types.ts b/src/lib/pushers/content-pusher/util/get-content-item-types.ts index 37fc1a8..c603ff9 100644 --- a/src/lib/pushers/content-pusher/util/get-content-item-types.ts +++ b/src/lib/pushers/content-pusher/util/get-content-item-types.ts @@ -14,6 +14,14 @@ import { collectContentIDReferences } from "./collect-content-id-references"; * reference (fullList=true) or a single-item reference (contentid/sortids). Linked * items are pushed FIRST so their source→target contentID mapping exists before the * referencing item's fields are remapped (PROD-2341). + * Linked levels: the linked items split into dependency levels, leaves first (PROD-3025). + * Level 0 references no other linked item; level n references something at level n-1. + * Each level is pushed as its own batch, so every mapping a level needs already exists. + * Items in a reference cycle share one level (their references to each other can't resolve). + * Linked cycle references: for each linked item in a reference cycle (or that references itself), + * its by-ID references into that cycle (PROD-3076). The pusher creates the item with those + * references blanked, then fills them in once every member has a target ID. Whole-list + * references resolve by container name, so they never block and are not included. * Skipped items: Items without valid container/model mappings */ export function getContentItemTypes( @@ -27,6 +35,8 @@ export function getContentItemTypes( ): { normalContentItems: ContentItem[]; linkedContentItems: ContentItem[]; + linkedContentLevels: ContentItem[][]; + linkedCycleReferences: Map; skippedItems: ContentItem[]; } { const { containerMapper, modelMapper } = opts; @@ -71,7 +81,14 @@ export function getContentItemTypes( // Build final result arrays const { normalContentItems, linkedContentItems } = buildResultArrays(normalSet, linkedSet, allItemsById); - return { normalContentItems, linkedContentItems, skippedItems: skipped }; + const { levels: linkedContentLevels, cycleReferences: linkedCycleReferences } = buildLinkedLevels( + linkedSet, + itemsByReferenceName, + allItemsById, + modelMapper + ); + + return { normalContentItems, linkedContentItems, linkedContentLevels, linkedCycleReferences, skippedItems: skipped }; } /** @@ -127,17 +144,24 @@ function collectReferencedContentIDs( } } - // Single-item references → the specific referenced item, if it is in this push set - const sourceModel = getSourceModel(item, modelMapper); - for (const contentID of collectContentIDReferences(item.fields || {}, sourceModel)) { - if (allItemsById.has(contentID)) { - ids.push(contentID); - } - } - + ids.push(...collectIdReferences(item, allItemsById, modelMapper)); return ids; } +/** + * The single-item (by contentID) references of an item that point at another item in this push + * set. Unlike a whole-list reference, each needs the referenced item's target ID before the item + * can be saved. + */ +function collectIdReferences( + item: ContentItem, + allItemsById: Map, + modelMapper: ModelMapper +): number[] { + const sourceModel = getSourceModel(item, modelMapper); + return collectContentIDReferences(item.fields || {}, sourceModel).filter((contentID) => allItemsById.has(contentID)); +} + /** * Resolves an item's own source model (by its definitionName), so a Content-typed field's * companion-field name (LinkeContentDropdownValueField/SortIDFieldName) can be read off it. @@ -194,6 +218,110 @@ function markReferencedItems( } } +/** + * Splits the linked items into dependency levels, leaves first (PROD-3025). + * + * Only references between linked items count: a reference to anything else is either already + * mapped or can't be resolved by ordering. Self-references are ignored. Cycles are collapsed + * with Tarjan's strongly-connected-components algorithm, so the levels come from a DAG and work + * at any depth. Tarjan emits a component only after every component it reaches, so each one's + * level is known from its dependencies' levels as it is emitted. Iterative, so a long chain + * can't overflow the call stack. Also returns each cycle member's by-ID references into its cycle. + */ +function buildLinkedLevels( + linkedSet: Set, + itemsByReferenceName: Map, + allItemsById: Map, + modelMapper: ModelMapper +): { levels: ContentItem[][]; cycleReferences: Map } { + const linkedIds = Array.from(linkedSet); + const cycleReferences = new Map(); + const depsById = new Map(); + for (const id of linkedIds) { + const deps = collectReferencedContentIDs(allItemsById.get(id)!, itemsByReferenceName, allItemsById, modelMapper); + depsById.set(id, Array.from(new Set(deps)).filter((dep) => dep !== id && linkedSet.has(dep))); + } + + const indexOf = new Map(); + const lowLink = new Map(); + const onStack = new Set(); + const componentStack: number[] = []; + const levelOf = new Map(); + let nextIndex = 0; + + const visit = (id: number) => { + indexOf.set(id, nextIndex); + lowLink.set(id, nextIndex); + nextIndex++; + componentStack.push(id); + onStack.add(id); + }; + + for (const root of linkedIds) { + if (indexOf.has(root)) continue; + visit(root); + const work: Array<{ id: number; next: number }> = [{ id: root, next: 0 }]; + + while (work.length > 0) { + const frame = work[work.length - 1]; + const deps = depsById.get(frame.id)!; + + if (frame.next < deps.length) { + const dep = deps[frame.next++]; + if (!indexOf.has(dep)) { + visit(dep); + work.push({ id: dep, next: 0 }); + } else if (onStack.has(dep)) { + lowLink.set(frame.id, Math.min(lowLink.get(frame.id)!, indexOf.get(dep)!)); + } + continue; + } + + work.pop(); + if (work.length > 0) { + const parent = work[work.length - 1]; + lowLink.set(parent.id, Math.min(lowLink.get(parent.id)!, lowLink.get(frame.id)!)); + } + if (lowLink.get(frame.id) !== indexOf.get(frame.id)) continue; + + // frame.id roots a component: pop its members, then place it one level above its deepest dependency + const members: number[] = []; + let member: number; + do { + member = componentStack.pop()!; + onStack.delete(member); + members.push(member); + } while (member !== frame.id); + + const memberSet = new Set(members); + let level = 0; + for (const id of members) { + for (const dep of depsById.get(id)!) { + if (!memberSet.has(dep)) level = Math.max(level, levelOf.get(dep)! + 1); + } + } + for (const id of members) levelOf.set(id, level); + + // PROD-3076: by-ID references inside the component (a self-reference counts) can't be + // resolved by ordering - the pusher fills them in with a second pass. + for (const id of members) { + const intoCycle = collectIdReferences(allItemsById.get(id)!, allItemsById, modelMapper).filter((dep) => + memberSet.has(dep) + ); + if (intoCycle.length > 0) cycleReferences.set(id, Array.from(new Set(intoCycle))); + } + } + } + + // Group in linkedSet order so the output is deterministic + const levels: ContentItem[][] = []; + for (const id of linkedIds) { + const level = levelOf.get(id)!; + (levels[level] ||= []).push(allItemsById.get(id)!); + } + return { levels: levels.filter(Boolean), cycleReferences }; +} + /** * Builds final arrays from ID sets, using the allItemsById map for lookup */ diff --git a/src/lib/pushers/content-pusher/util/strip-content-references.ts b/src/lib/pushers/content-pusher/util/strip-content-references.ts new file mode 100644 index 0000000..b009f43 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/strip-content-references.ts @@ -0,0 +1,58 @@ +import { getLinkedContentCompanionFields, findFieldKey } from "lib/content/linked-content-companion-fields"; + +/** + * PROD-3076: copy of an item's fields with its references to the given source contentIDs blanked, + * so the item can be created before the items it references exist (pass 1 of a reference cycle). + * + * Covers the same reference shapes as collectContentIDReferences: + * - `contentid` / `contentID` set to -1 ("no reference selected", which the field mapper and the + * PROD-2309 guard both treat as empty; the server stores the field empty, even a required one); + * - each ID removed from a comma-separated `sortids`; + * - each ID removed from a companion field (the model's LinkeContentDropdownValueField / + * SortIDFieldName, PROD-2446), a top-level sibling of the main field. + * + * Returns the new fields and how many references were blanked. The input is never modified. + */ +export function stripContentReferences( + fields: any, + ids: Set, + model?: any +): { fields: any; stripped: number } { + let stripped = 0; + + const withoutIds = (list: string): string => { + const parts = list.split(",").map((p) => p.trim()); + const kept = parts.filter((p) => !ids.has(parseInt(p))); + stripped += parts.length - kept.length; + return kept.join(","); + }; + + function walk(node: any): any { + if (Array.isArray(node)) return node.map(walk); + if (!node || typeof node !== "object") return node; + + const out: any = {}; + for (const [key, value] of Object.entries(node)) { + if ((key === "contentid" || key === "contentID") && typeof value === "number" && ids.has(value)) { + out[key] = -1; + stripped++; + } else if (key === "sortids" && typeof value === "string") { + out[key] = withoutIds(value); + } else { + out[key] = walk(value); + } + } + return out; + } + + const result = walk(fields || {}); + + for (const [, companionFieldName] of getLinkedContentCompanionFields(model)) { + const companionKey = findFieldKey(result, companionFieldName); + if (companionKey && typeof result[companionKey] === "string" && result[companionKey].trim()) { + result[companionKey] = withoutIds(result[companionKey]); + } + } + + return { fields: result, stripped }; +} diff --git a/src/lib/pushers/content-pusher/util/tests/change-detection.test.ts b/src/lib/pushers/content-pusher/util/tests/change-detection.test.ts index c51431f..9e26073 100644 --- a/src/lib/pushers/content-pusher/util/tests/change-detection.test.ts +++ b/src/lib/pushers/content-pusher/util/tests/change-detection.test.ts @@ -103,13 +103,38 @@ describe("changeDetection — conflict path", () => { expect(result.isConflict).toBe(false); }); - it("does NOT conflict when only target version increased", () => { + it("conflicts when only the target version increased (an edit on the target)", () => { setState({ sourceGuid: "src-guid", targetGuid: "tgt-guid" }); const source = makeContent(1, 5); const target = makeContent(100, 10); const mapping = makeMapping({ sourceVersionID: 5, targetVersionID: 5 }); const result = changeDetection(source, target, mapping, "en-us"); + expect(result.isConflict).toBe(true); + expect(result.shouldUpdate).toBe(false); + expect(result.shouldSkip).toBe(false); + expect(result.reason).toMatch(/target changed since the last sync and the source didn't/); + expect(result.reason).toMatch(/--overwrite/); + }); + + it("replaces a target-only edit with the source under --overwrite", () => { + setState({ overwrite: true, sourceGuid: "src-guid", targetGuid: "tgt-guid" }); + const source = makeContent(1, 5); + const target = makeContent(100, 10); + const mapping = makeMapping({ sourceVersionID: 5, targetVersionID: 5 }); + const result = changeDetection(source, target, mapping, "en-us"); expect(result.isConflict).toBe(false); + expect(result.shouldUpdate).toBe(true); + expect(result.entity).toBe(target); + }); + + it("leaves a mapping with no recorded target version alone (older mapping files)", () => { + setState({ sourceGuid: "src-guid", targetGuid: "tgt-guid" }); + const source = makeContent(1, 5); + const target = makeContent(100, 10); + const mapping = makeMapping({ sourceVersionID: 5, targetVersionID: 0 }); + const result = changeDetection(source, target, mapping, "en-us"); + expect(result.isConflict).toBe(false); + expect(result.shouldSkip).toBe(true); }); }); diff --git a/src/lib/pushers/content-pusher/util/tests/get-content-item-types.test.ts b/src/lib/pushers/content-pusher/util/tests/get-content-item-types.test.ts index 7bd2c80..ffc3955 100644 --- a/src/lib/pushers/content-pusher/util/tests/get-content-item-types.test.ts +++ b/src/lib/pushers/content-pusher/util/tests/get-content-item-types.test.ts @@ -375,3 +375,179 @@ describe("getContentItemTypes — recursive references", () => { expect(() => getContentItemTypes([itemA, itemB], makeValidOpts())).not.toThrow(); }); }); + +// ─── dependency levels (PROD-3025) ─────────────────────────────────────────── + +describe("getContentItemTypes — linked dependency levels (PROD-3025)", () => { + const idsByLevel = (levels: any[][]) => levels.map((level) => level.map((i) => i.contentID)); + + it("splits a three-deep chain so the leaf is pushed before the item that references it", () => { + // FeaturedPost → Post → Category: Post and Category are both linked, but Category must go first + const category = makeItem("categories", "Category"); + const post = makeItem("posts", "Post", { category: { contentid: category.contentID } }); + const featured = makeItem("featuredposts", "FeaturedPost", { post: { contentid: post.contentID } }); + + const result = getContentItemTypes([featured, post, category], makeValidOpts()); + + expect(result.normalContentItems).toEqual([featured]); + expect(idsByLevel(result.linkedContentLevels)).toEqual([[category.contentID], [post.contentID]]); + }); + + it("gives every item of a 50-deep chain its own level, leaves first", () => { + const chain: any[] = [makeItem("item-0", "Model")]; + for (let i = 1; i < 50; i++) { + chain.push(makeItem(`item-${i}`, "Model", { next: { contentid: chain[i - 1].contentID } })); + } + + const result = getContentItemTypes([...chain].reverse(), makeValidOpts()); + + // The last item is unreferenced (normal); the other 49 form 49 levels in chain order + expect(result.normalContentItems).toEqual([chain[49]]); + expect(idsByLevel(result.linkedContentLevels)).toEqual(chain.slice(0, 49).map((i) => [i.contentID])); + }); + + it("places an item one level above its deepest dependency when depths differ", () => { + const leaf = makeItem("leaf", "Model"); + const mid = makeItem("mid", "Model", { leaf: { contentid: leaf.contentID } }); + // uneven depends on mid (level 1) and directly on leaf (level 0) → level 2 + const uneven = makeItem("uneven", "Model", { + mid: { contentid: mid.contentID }, + leaf: { contentid: leaf.contentID }, + }); + const top = makeItem("top", "Model", { uneven: { contentid: uneven.contentID } }); + + const result = getContentItemTypes([top, uneven, mid, leaf], makeValidOpts()); + + expect(idsByLevel(result.linkedContentLevels)).toEqual([[leaf.contentID], [mid.contentID], [uneven.contentID]]); + }); + + it("puts both sides of a diamond on the same level, above their shared dependency", () => { + const shared = makeItem("shared", "Model"); + const left = makeItem("left", "Model", { shared: { contentid: shared.contentID } }); + const right = makeItem("right", "Model", { shared: { contentid: shared.contentID } }); + const top = makeItem("top", "Model", { picks: { sortids: `${left.contentID},${right.contentID}` } }); + + const result = getContentItemTypes([top, left, right, shared], makeValidOpts()); + + expect(result.linkedContentLevels).toHaveLength(2); + expect(idsByLevel(result.linkedContentLevels)[0]).toEqual([shared.contentID]); + expect(idsByLevel(result.linkedContentLevels)[1].sort()).toEqual([left.contentID, right.contentID].sort()); + }); + + it("orders whole-list references the same way", () => { + const tag = makeItem("tags", "Tag"); + const post = makeItem("posts", "Post", { tags: { referencename: "tags", fulllist: true } }); + const page = makeItem("listing", "Listing", { posts: { referencename: "posts", fulllist: true } }); + + const result = getContentItemTypes([page, post, tag], makeValidOpts()); + + expect(idsByLevel(result.linkedContentLevels)).toEqual([[tag.contentID], [post.contentID]]); + }); + + it("keeps a cycle on one level, after what it depends on, and the item above it on the next", () => { + const leaf = makeItem("leaf", "Model"); + const a = makeItem("a", "Model", { b: { contentid: 0 }, leaf: { contentid: leaf.contentID } }); + const b = makeItem("b", "Model", { a: { contentid: a.contentID } }); + a.fields.b.contentid = b.contentID; + const top = makeItem("top", "Model", { a: { contentid: a.contentID } }); + + const result = getContentItemTypes([top, a, b, leaf], makeValidOpts()); + + expect(result.linkedContentLevels).toHaveLength(2); + expect(idsByLevel(result.linkedContentLevels)[0]).toEqual([leaf.contentID]); + expect(idsByLevel(result.linkedContentLevels)[1].sort()).toEqual([a.contentID, b.contentID].sort()); + }); + + it("ignores a self-reference when placing an item", () => { + const child = makeItem("child", "Model"); + const self = makeItem("self", "Model", { child: { contentid: child.contentID } }); + self.fields.me = { contentid: self.contentID }; + const parent = makeItem("parent", "Model", { self: { contentid: self.contentID } }); + + const result = getContentItemTypes([parent, self, child], makeValidOpts()); + + expect(idsByLevel(result.linkedContentLevels)).toEqual([[child.contentID], [self.contentID]]); + }); + + it("covers exactly the linked items", () => { + const category = makeItem("categories", "Category"); + const post = makeItem("posts", "Post", { category: { contentid: category.contentID } }); + const outside = makeItem("outside", "Model", { missing: { contentid: 9999 } }); + const featured = makeItem("featuredposts", "FeaturedPost", { post: { contentid: post.contentID } }); + + const result = getContentItemTypes([featured, post, category, outside], makeValidOpts()); + + const flattened = result.linkedContentLevels.flat().map((i) => i.contentID).sort(); + expect(flattened).toEqual(result.linkedContentItems.map((i) => i.contentID).sort()); + }); +}); + +// ─── reference cycles (PROD-3076) ──────────────────────────────────────────── + +describe("getContentItemTypes — linked cycle references (PROD-3076)", () => { + it("reports both sides of a two-item cycle with the reference into it", () => { + const a = makeItem("nodes", "Node"); + const b = makeItem("nodes", "Node", { next: { contentid: a.contentID } }); + a.fields = { next: { contentid: b.contentID } }; + const top = makeItem("top", "Model", { node: { contentid: a.contentID } }); + + const { linkedCycleReferences } = getContentItemTypes([top, a, b], makeValidOpts()); + + expect(linkedCycleReferences.get(a.contentID)).toEqual([b.contentID]); + expect(linkedCycleReferences.get(b.contentID)).toEqual([a.contentID]); + expect(linkedCycleReferences.has(top.contentID)).toBe(false); + }); + + it("reports a self-reference", () => { + const self = makeItem("nodes", "Node"); + self.fields = { next: { contentid: self.contentID } }; + + const { linkedCycleReferences } = getContentItemTypes([self], makeValidOpts()); + + expect(linkedCycleReferences.get(self.contentID)).toEqual([self.contentID]); + }); + + it("covers a three-item cycle and not the leaf it depends on", () => { + const leaf = makeItem("leaf", "Model"); + const a = makeItem("a", "Model"); + const b = makeItem("b", "Model"); + const c = makeItem("c", "Model"); + a.fields = { next: { contentid: b.contentID }, leaf: { contentid: leaf.contentID } }; + b.fields = { next: { contentid: c.contentID } }; + c.fields = { next: { contentid: a.contentID } }; + + const { linkedCycleReferences } = getContentItemTypes([a, b, c, leaf], makeValidOpts()); + + expect(linkedCycleReferences.get(a.contentID)).toEqual([b.contentID]); + expect(linkedCycleReferences.get(b.contentID)).toEqual([c.contentID]); + expect(linkedCycleReferences.get(c.contentID)).toEqual([a.contentID]); + expect(linkedCycleReferences.has(leaf.contentID)).toBe(false); + }); + + it("ignores a cycle made only of whole-list references, which never block", () => { + const a = makeItem("list-a", "ModelA", { other: { referencename: "list-b", fulllist: true } }); + const b = makeItem("list-b", "ModelB", { other: { referencename: "list-a", fulllist: true } }); + + const { linkedCycleReferences } = getContentItemTypes([a, b], makeValidOpts()); + + expect(linkedCycleReferences.size).toBe(0); + }); + + it("reports only the by-ID side of a cycle closed by a whole-list reference", () => { + const b = makeItem("list-b", "ModelB", { back: { referencename: "list-a", fulllist: true } }); + const a = makeItem("list-a", "ModelA", { pick: { contentid: b.contentID } }); + + const { linkedCycleReferences } = getContentItemTypes([a, b], makeValidOpts()); + + expect(linkedCycleReferences.get(a.contentID)).toEqual([b.contentID]); + expect(linkedCycleReferences.has(b.contentID)).toBe(false); + }); + + it("reports nothing for a plain chain", () => { + const leaf = makeItem("leaf", "Model"); + const mid = makeItem("mid", "Model", { leaf: { contentid: leaf.contentID } }); + const top = makeItem("top", "Model", { mid: { contentid: mid.contentID } }); + + expect(getContentItemTypes([top, mid, leaf], makeValidOpts()).linkedCycleReferences.size).toBe(0); + }); +}); diff --git a/src/lib/pushers/content-pusher/util/tests/strip-content-references.test.ts b/src/lib/pushers/content-pusher/util/tests/strip-content-references.test.ts new file mode 100644 index 0000000..999d035 --- /dev/null +++ b/src/lib/pushers/content-pusher/util/tests/strip-content-references.test.ts @@ -0,0 +1,52 @@ +import { stripContentReferences } from "../strip-content-references"; + +// PROD-3076: pass 1 of a reference cycle blanks only the references to the given source IDs. + +describe("stripContentReferences", () => { + it("sets a single-item reference to -1 and leaves others alone", () => { + const fields = { next: { contentid: 5, fulllist: false }, other: { contentid: 9, fulllist: false } }; + const { fields: out, stripped } = stripContentReferences(fields, new Set([5])); + expect(out).toEqual({ next: { contentid: -1, fulllist: false }, other: { contentid: 9, fulllist: false } }); + expect(stripped).toBe(1); + }); + + it("handles contentID spelled with a capital D", () => { + const { fields } = stripContentReferences({ link: { contentID: 5 } }, new Set([5])); + expect(fields.link.contentID).toBe(-1); + }); + + it("removes only the given IDs from sortids", () => { + const { fields, stripped } = stripContentReferences({ picks: { sortids: "3, 5,7" } }, new Set([5, 7])); + expect(fields.picks.sortids).toBe("3"); + expect(stripped).toBe(2); + }); + + it("finds references nested in arrays and objects", () => { + const fields = { rows: [{ cell: { contentid: 5 } }, { cell: { contentid: 6 } }] }; + const { fields: out, stripped } = stripContentReferences(fields, new Set([5, 6])); + expect(out.rows.map((r: any) => r.cell.contentid)).toEqual([-1, -1]); + expect(stripped).toBe(2); + }); + + it("removes the IDs from a companion field named by the model (PROD-2446)", () => { + const model = { fields: [{ name: "Next", settings: { LinkeContentDropdownValueField: "NextID" } }] }; + const fields = { next: { contentid: 5, fulllist: false }, nextID: "5" }; + const { fields: out, stripped } = stripContentReferences(fields, new Set([5]), model); + expect(out).toEqual({ next: { contentid: -1, fulllist: false }, nextID: "" }); + expect(stripped).toBe(2); + }); + + it("does not touch whole-list references or other values", () => { + const fields = { list: { referencename: "posts", fulllist: true }, title: "5", count: 5 }; + const { fields: out, stripped } = stripContentReferences(fields, new Set([5])); + expect(out).toEqual(fields); + expect(stripped).toBe(0); + }); + + it("never modifies the input", () => { + const fields = { next: { contentid: 5 }, picks: { sortids: "5,6" } }; + const before = JSON.stringify(fields); + stripContentReferences(fields, new Set([5])); + expect(JSON.stringify(fields)).toBe(before); + }); +}); diff --git a/src/lib/pushers/content-pusher/util/types.ts b/src/lib/pushers/content-pusher/util/types.ts index 39c057c..994bea7 100644 --- a/src/lib/pushers/content-pusher/util/types.ts +++ b/src/lib/pushers/content-pusher/util/types.ts @@ -15,6 +15,9 @@ export interface ContentBatchConfig { defaultAssetUrl?: string; // Default asset URL for content mapping targetData?: any; // Target instance data for checking existing content onBatchComplete?: (batchResult: BatchProcessingResult, batchNumber: number) => Promise; // Callback after each batch completes + // PROD-3076: update the target item a mapping points at even though it isn't in the pulled target + // data, because it was created earlier in this run (pass 2 of a reference cycle). + trustSameRunMappings?: boolean; } /** diff --git a/src/lib/pushers/gallery-pusher.ts b/src/lib/pushers/gallery-pusher.ts index 16adb5f..22573db 100644 --- a/src/lib/pushers/gallery-pusher.ts +++ b/src/lib/pushers/gallery-pusher.ts @@ -101,15 +101,12 @@ export async function pushGalleries( successful++; } else if (existingMapping) { const targetGallery = targetGalleryById || targetGalleryByName; - const isTargetSafe = referenceMapper.hasTargetChanged(targetGallery); + const hasTargetChanges = referenceMapper.hasTargetChanged(targetGallery); const hasSourceChanges = referenceMapper.hasSourceChanged(sourceGallery); - let shouldUpdate = isTargetSafe && hasSourceChanges; - let shouldSkip = !isTargetSafe && !hasSourceChanges; - - if (overwrite) { - shouldUpdate = true; - shouldSkip = false; - } + // Source changed alone: update. Target changed (alone or with the source): a conflict, + // replaced only with --overwrite. Neither: up to date. + const shouldUpdate = (hasSourceChanges && !hasTargetChanges) || (overwrite && hasTargetChanges); + const shouldSkip = !hasSourceChanges && !hasTargetChanges; if (shouldUpdate) { // Gallery exists but needs updating @@ -137,9 +134,13 @@ export async function pushGalleries( }); skipped++; } else { - // Neither update nor skip: target changed while source also changed and - // --overwrite was not set. The real sync silently does nothing here; in - // preflight we surface it as a conflict so it isn't missed. + // Conflict: the target changed since the last sync and --overwrite was not set. + logger.gallery.skipped( + sourceGallery, + "conflict: target changed since the last sync; use --overwrite to replace it", + targetGuid + ); + skipped++; preflightReport.record({ phase: "Galleries", action: "conflict", diff --git a/src/lib/pushers/orchestrate-pushers.ts b/src/lib/pushers/orchestrate-pushers.ts index 57b49a4..32c62c2 100644 --- a/src/lib/pushers/orchestrate-pushers.ts +++ b/src/lib/pushers/orchestrate-pushers.ts @@ -226,6 +226,17 @@ export class Pushers { (state.models && state.models.trim().length > 0) || (state.modelsWithDeps && state.modelsWithDeps.trim().length > 0); + // --syncDeletes decides "deleted in the source" from what is absent in the source. A scoped run + // narrows the source to part of the instance, so everything outside the scope would look + // deleted. Refuse the combination before anything is written. + if (state.syncDeletes && (pageSelectors.length > 0 || containerSelectors.length > 0 || hasModelFilter)) { + throw new Error( + "Sync deletes validation failed. --syncDeletes cannot be combined with --pages, --containers, --models " + + "or --models-with-deps — a scoped run cannot tell a deleted entity from one outside the scope. " + + "Run the deletes as a separate, unscoped sync (use --elements to limit the entity types)." + ); + } + if (pageSelectors.length > 0 && containerSelectors.length > 0) { throw new Error( "Page validation failed. --pages cannot be combined with --containers — each defines a " + @@ -335,6 +346,10 @@ export class Pushers { console.error(ansiColors.yellow(`Warning during guid-level operations: ${error?.message || error}`)); } + // List order (PROD-2992) only touches a scoped run's lists, so keep the content each locale pushed. + const isScoped = Object.keys(filterOptions).length > 0; + const scopedContent = new Map(); + // Do the locale level ops try { for (const config of pusherConfig) { @@ -346,6 +361,7 @@ export class Pushers { Object.keys(filterOptions).length > 0 ? filterOptions : undefined ); const targetData = await targetDataLoader.loadGuidEntities(locale); + if (config === PUSH_OPERATIONS.content && isScoped) scopedContent.set(locale, sourceData.content || []); // Track IDs for this specific locale const localeContentIds: number[] = []; @@ -386,6 +402,59 @@ export class Pushers { } } + // PROD-3042: a page save can bump the modified date of its components' containers. Advance the + // mappings of containers that were in sync before this run, so the next run doesn't see them + // as changed on the target. + if (phases.some((p) => p.operation === "pushPages" && p.successful > 0)) { + const { refreshContainerDatesAfterPageSaves } = await import("../mappers/refresh-container-dates"); + await refreshContainerDatesAfterPageSaves(sourceGuid, targetGuid, locales[0]); + } + + // --syncDeletes runs last: every push has repointed what it could, so the targets that are + // still only reachable from a deleted source can go, dependents first. + if (state.syncDeletes) { + const { runSyncDeletes } = await import("../deleters/sync-deletes"); + const deletes = await runSyncDeletes({ sourceGuid, targetGuid, locales, elements }); + totalSuccess += deletes.successful; + totalFailures += deletes.failed; + totalSkipped += deletes.skipped; + failureDetails.push(...(deletes.failureDetails || [])); + warningDetails.push(...(deletes.warningDetails || [])); + phases.push({ + operation: "syncDeletes", + description: "Delete entities deleted in the source", + successful: deletes.successful, + failed: deletes.failed, + skipped: deletes.skipped, + status: deletes.status === "error" ? "error" : "success", + }); + } + + // List order runs after content and deletes, and before auto-publish (which copies each + // item's order into its new published version). A list is changed only when its order + // differs from the source's. + if (elements.includes("Content")) { + const { runSyncListOrder } = await import("../list-order/sync-list-order"); + const order = await runSyncListOrder( + { sourceGuid, targetGuid, locales }, + isScoped ? scopedContent : undefined, + publishableContentIdsByLocale + ); + totalSuccess += order.successful; + totalFailures += order.failed; + totalSkipped += order.skipped; + failureDetails.push(...(order.failureDetails || [])); + warningDetails.push(...(order.warningDetails || [])); + phases.push({ + operation: "listOrder", + description: "Put synced content lists in the source's item order", + successful: order.successful, + failed: order.failed, + skipped: order.skipped, + status: order.status === "error" ? "error" : "success", + }); + } + return { totalSuccess, totalFailures, diff --git a/src/lib/pushers/page-pusher/process-page.ts b/src/lib/pushers/page-pusher/process-page.ts index 927c19d..ea4363b 100644 --- a/src/lib/pushers/page-pusher/process-page.ts +++ b/src/lib/pushers/page-pusher/process-page.ts @@ -390,9 +390,11 @@ export async function processPage({ // Fix zones format - ensure zones is always a defined object (never null/undefined) const formattedZones = mappedZones && typeof mappedZones === "object" ? mappedZones : {}; - // CRITICAL FIX: Ensure every page has a valid title field - // Folder pages often don't have titles, but API requires them - const pageTitle = page.title || page.menuText || page.name || "Untitled Page"; + // Send the source title as-is. The Fetch API (Content Sync) returns an empty title as null, and + // the Management API accepts a missing or empty title (folders ignore it), so null goes as "". + // Filling in the menu text or page name made an untitled page - typically a dynamic page, whose + // titles come from its titleFormula - differ from the source on every sync (PROD-3025, PROD-3043). + const pageTitle = page.title ?? ""; const pageJSON = JSON.stringify(page, null, 2); const pageCopy = JSON.parse(pageJSON) as mgmtApi.PageItem; // Create a copy to avoid modifying original diff --git a/src/lib/pushers/page-pusher/tests/process-page.test.ts b/src/lib/pushers/page-pusher/tests/process-page.test.ts index d1af91b..01ae18a 100644 --- a/src/lib/pushers/page-pusher/tests/process-page.test.ts +++ b/src/lib/pushers/page-pusher/tests/process-page.test.ts @@ -328,6 +328,46 @@ describe("processPage — folder pages", () => { }); }); +// ─── page title in the save payload (PROD-3025) ────────────────────────────── + +describe("processPage — page title sent to the target (PROD-3025)", () => { + async function savedTitle(page: any): Promise { + const { TemplateMapper } = require("lib/mappers/template-mapper"); + TemplateMapper.mockImplementation(() => ({ + getTemplateMappingByPageTemplateName: jest.fn().mockReturnValue({ ref: "Main" }), + getMappedEntity: jest.fn().mockReturnValue({ contentSectionDefinitions: [] }), + })); + const apiClient = makeApiClient(); + await processPage(makeProps({ page, apiClient, pageMapper: makePageMapper() })); + expect(apiClient.pageMethods.savePage).toHaveBeenCalled(); + return apiClient.pageMethods.savePage.mock.calls[0][0].title; + } + + it("keeps an untitled dynamic page untitled instead of filling in its name", async () => { + // As pulled through the Fetch API: an empty title comes back as null + const page = makePage({ + name: "posts-dynamic", + pageType: "dynamic", + title: null, + menuText: "", + dynamic: { referenceName: "Posts", titleFormula: "##Title##", pageNameFormula: "##Slug##" }, + }); + expect(await savedTitle(page)).toBe(""); + }); + + it("keeps an empty title empty", async () => { + expect(await savedTitle(makePage({ title: "", menuText: "Blog" }))).toBe(""); + }); + + it("sends a page's own title unchanged", async () => { + expect(await savedTitle(makePage({ title: "About Us" }))).toBe("About Us"); + }); + + it("does not take the menu text or name for a page with no title", async () => { + expect(await savedTitle(makePage({ title: undefined, menuText: "Blog", name: "blog" }))).toBe(""); + }); +}); + // ─── successful save via batch ───────────────────────────────────────────────── describe("processPage — successful batch save", () => { @@ -353,6 +393,33 @@ describe("processPage — successful batch save", () => { expect(result.status).toBe("success"); }); + describe("title (PROD-3043)", () => { + function savedTitle(page: any): Promise { + const { TemplateMapper } = require("lib/mappers/template-mapper"); + TemplateMapper.mockImplementation(() => ({ + getTemplateMappingByPageTemplateName: jest.fn().mockReturnValue({ ref: "Main" }), + getMappedEntity: jest.fn().mockReturnValue({ contentSectionDefinitions: [] }), + })); + mockPoll.mockResolvedValue({ failedItems: [], successItems: [makePage({ pageID: 200 })] }); + mockExtract.mockReturnValue({ successfulItems: [{ newId: 200, newItem: { processedItemVersionID: 5 } }], failedItems: [] }); + const props = makeProps({ page, pageMapper: makePageMapper({ hasSourceChanged: jest.fn().mockReturnValue(true) }) }); + return processPage(props).then(() => props.apiClient.pageMethods.savePage.mock.calls[0][0].title); + } + + it("sends a dynamic page's empty title as empty, not its name", async () => { + await expect(savedTitle(makePage({ pageType: "dynamic", name: "posts-dynamic", title: null, menuText: "Posts" }))).resolves.toBe(""); + }); + + it("sends a dynamic page's own title when it has one", async () => { + await expect(savedTitle(makePage({ pageType: "dynamic", name: "posts-dynamic", title: "Posts" }))).resolves.toBe("Posts"); + }); + + it("keeps a static page without a title untitled too, rather than taking its menu text or name", async () => { + await expect(savedTitle(makePage({ title: null, menuText: "Menu" }))).resolves.toBe(""); + await expect(savedTitle(makePage({ title: "", menuText: null, name: "about" }))).resolves.toBe(""); + }); + }); + it("calls pageMapper.addMapping after a successful save", async () => { const { TemplateMapper } = require("lib/mappers/template-mapper"); TemplateMapper.mockImplementation(() => ({ diff --git a/src/lib/pushers/tests/asset-pusher-stale-mapping.test.ts b/src/lib/pushers/tests/asset-pusher-stale-mapping.test.ts index 9930690..d0651cc 100644 --- a/src/lib/pushers/tests/asset-pusher-stale-mapping.test.ts +++ b/src/lib/pushers/tests/asset-pusher-stale-mapping.test.ts @@ -281,6 +281,51 @@ describe("pushAssets — asset moved to another folder (PROD-2627)", () => { }); }); +// ─── an edit on the target only ─────────────────────────────────────────────── + +describe("pushAssets — mapped asset changed on the target only", () => { + it("is a conflict without --overwrite: the target's version is kept, nothing uploaded", async () => { + new AssetMapper(SRC, TGT).addMapping(asset("source", 1, "a/x.png", OLD), asset("target", 2, "a/x.png", OLD)); + writeLocalFile("a/x.png"); + mockApi(); + + const { pushAssets } = await import("../asset-pusher"); + const result = await pushAssets([asset("source", 1, "a/x.png", OLD)], [asset("target", 2, "a/x.png", NEW)]); + + expect(axios.post).not.toHaveBeenCalled(); + expect(result.skipped).toBe(1); + expect(result.successful).toBe(0); + }); + + it("is reported as a preflight conflict", async () => { + // seed before enabling preflight: preflight never persists mapping files + new AssetMapper(SRC, TGT).addMapping(asset("source", 1, "a/x.png", OLD), asset("target", 2, "a/x.png", OLD)); + state.preflight = true; + const { preflightReport } = await import("lib/preflight/preflight-report"); + preflightReport.reset(); + mockApi(); + + const { pushAssets } = await import("../asset-pusher"); + await pushAssets([asset("source", 1, "a/x.png", OLD)], [asset("target", 2, "a/x.png", NEW)]); + + expect(await preflightEntries()).toEqual([expect.objectContaining({ action: "conflict", name: "x.png" })]); + }); + + it("with --overwrite replaces it with the source", async () => { + setState({ overwrite: true }); + new AssetMapper(SRC, TGT).addMapping(asset("source", 1, "a/x.png", OLD), asset("target", 2, "a/x.png", OLD)); + writeLocalFile("a/x.png"); + mockApi(); + axios.post.mockResolvedValue({ data: [asset("target", 2, "a/x.png", NEW)] }); + + const { pushAssets } = await import("../asset-pusher"); + const result = await pushAssets([asset("source", 1, "a/x.png", OLD)], [asset("target", 2, "a/x.png", NEW)]); + + expect(axios.post).toHaveBeenCalledTimes(1); + expect(result.successful).toBe(1); + }); +}); + // ─── duplicate records from earlier versions ────────────────────────────────── describe("pushAssets — duplicate records left by earlier versions (PROD-2627)", () => { diff --git a/src/lib/pushers/tests/gallery-pusher.test.ts b/src/lib/pushers/tests/gallery-pusher.test.ts index 57955e2..1f775e8 100644 --- a/src/lib/pushers/tests/gallery-pusher.test.ts +++ b/src/lib/pushers/tests/gallery-pusher.test.ts @@ -164,3 +164,47 @@ describe("pushGalleries — result shape", () => { expect(result).toHaveProperty("skipped"); }); }); + +// ─── pushGalleries — change detection ───────────────────────────────────────── + +describe("pushGalleries — mapped gallery: what changed decides update, skip or conflict", () => { + async function run(sourceChanged: boolean, targetChanged: boolean, overwrite = false) { + const { GalleryMapper } = await import("lib/mappers/gallery-mapper"); + const source = makeGallery({ name: "Mapped Gallery" }); + const target = makeGallery({ name: "Mapped Gallery", mediaGroupingID: 500 }); + jest.spyOn(GalleryMapper.prototype, "getGalleryMapping").mockReturnValue({ targetMediaGroupingID: 500 } as any); + jest.spyOn(GalleryMapper.prototype, "hasSourceChanged").mockReturnValue(sourceChanged); + jest.spyOn(GalleryMapper.prototype, "hasTargetChanged").mockReturnValue(targetChanged); + jest.spyOn(GalleryMapper.prototype, "addMapping").mockImplementation(() => {}); + const saveMock = jest.fn().mockResolvedValue(target); + jest.spyOn(stateModule, "getApiClient").mockReturnValue(makeApiClient(saveMock)); + setState({ overwrite }); + + const { pushGalleries } = await import("../gallery-pusher"); + const result = await pushGalleries([source], [target]); + return { result, saved: saveMock.mock.calls.length }; + } + + it("updates when only the source changed", async () => { + const { result, saved } = await run(true, false); + expect(saved).toBe(1); + expect(result.successful).toBe(1); + }); + + it("skips when neither changed, even with --overwrite", async () => { + expect((await run(false, false)).saved).toBe(0); + expect((await run(false, false, true)).saved).toBe(0); + }); + + it("keeps a target-only edit (a conflict) unless --overwrite", async () => { + const kept = await run(false, true); + expect(kept.saved).toBe(0); + expect(kept.result.skipped).toBe(1); + expect((await run(false, true, true)).saved).toBe(1); + }); + + it("keeps an edit on both sides (a conflict) unless --overwrite", async () => { + expect((await run(true, true)).saved).toBe(0); + expect((await run(true, true, true)).saved).toBe(1); + }); +}); diff --git a/src/lib/verify/tests/check-content.test.ts b/src/lib/verify/tests/check-content.test.ts index 863e9e6..073b8ea 100644 --- a/src/lib/verify/tests/check-content.test.ts +++ b/src/lib/verify/tests/check-content.test.ts @@ -73,6 +73,40 @@ describe("checkContent", () => { expect(collector.findings).toEqual([]); }); + it("treats links to an item deleted on both sides as equal", () => { + // Source item 2 and its target copy 102 were deleted (by --syncDeletes, which also drops the + // mapping); 1 and 101 still point at them. + const collector = run( + [item(1, { title: "A", related: { contentid: 2, fulllist: false } })], + [item(101, { title: "A", related: "102" })], + [[1, 101]] + ); + expect(collector.findings).toEqual([]); + }); + + it("still fails a link to an unmapped item that exists in the source", () => { + const collector = run( + [item(1, { title: "A", related: { contentid: 2, fulllist: false } }), item(2, { title: "B" })], + [item(101, { title: "A", related: "102" })], + [[1, 101]] + ); + expect(collector.findings).toEqual( + expect.arrayContaining([expect.objectContaining({ category: "fieldMismatch", path: "related", sourceID: 1 })]) + ); + }); + + it("keeps a mapped target ID that is missing from the pull (e.g. unpublished)", () => { + const collector = run( + [item(1, { title: "A", related: { contentid: 2, fulllist: false } })], + [item(101, { title: "A", related: "102" })], + [ + [1, 101], + [2, 102], + ] + ); + expect(collector.findings.filter((f) => f.path === "related")).toEqual([]); + }); + it("warns, without failing, on a publish-state difference", () => { const collector = run( [item(1, { title: "A" }, { state: 2 })], diff --git a/src/lib/verify/verify-context.ts b/src/lib/verify/verify-context.ts index d6742cf..5e74580 100644 --- a/src/lib/verify/verify-context.ts +++ b/src/lib/verify/verify-context.ts @@ -220,13 +220,21 @@ export function createVerifyContext(input: { }; } +/** + * A content reference to an item that no longer exists on its side and has no mapping, e.g. both + * halves of a pair `--syncDeletes` removed. The same token on both sides, so two such references + * compare equal instead of `unmapped:` against a bare target ID. + */ +const DANGLING_CONTENT_REF = "dangling"; + /** Canonicalization context for SOURCE values: every ID is translated to its target. */ export function sourceCanonicalContext(ctx: VerifyContext): CanonicalContext { return { side: "source", contentRef: (id) => { const target = ctx.localeMappings.content.target(id); - return target !== undefined ? String(target) : `unmapped:${id}`; + if (target !== undefined) return String(target); + return ctx.source.contentById.has(id) ? `unmapped:${id}` : DANGLING_CONTENT_REF; }, containerRef: (name) => { const key = lower(name); @@ -252,7 +260,10 @@ export function sourceCanonicalContext(ctx: VerifyContext): CanonicalContext { export function targetCanonicalContext(ctx: VerifyContext): CanonicalContext { return { side: "target", - contentRef: (id) => String(id), + // Not on the target and no mapping points at it: deleted, like its source counterpart. (A + // mapped item missing from the pull, such as an unpublished one, keeps its ID.) + contentRef: (id) => + ctx.target.contentById.has(id) || ctx.localeMappings.content.hasTarget(id) ? String(id) : DANGLING_CONTENT_REF, containerRef: (name) => lower(name), galleryRef: (id) => String(id), isEmptyList: (name) => !ctx.target.contentCountByContainer.get(lower(name)), diff --git a/src/tests/sync/full-instance.test.ts b/src/tests/sync/full-instance.test.ts index a545b49..e6550b2 100644 --- a/src/tests/sync/full-instance.test.ts +++ b/src/tests/sync/full-instance.test.ts @@ -15,6 +15,10 @@ import { SyncHarness } from "./helpers/run-sync"; import { installNetworkStubs, InstalledNetworkStubs } from "./helpers/network-stubs"; import { ALL_ELEMENTS, ALL_OPERATIONS, FIXTURE_LOCALE, fullInstanceFixture, MODEL_POST } from "./helpers/full-instance-fixture"; +// The post-push list order phase (PROD-2992) runs after the eight push phases; it's asserted on its own. +const LIST_ORDER = "listOrder"; +const pushPhases = (phases: T[]) => phases.filter((p) => p.operation !== LIST_ORDER); + jest.mock("lib/pushers/batch-polling", () => ({ pollBatchUntilComplete: jest.fn(), extractContentBatchResults: jest.fn(), @@ -61,8 +65,19 @@ describe("full instance — all eight phases", () => { // A phase missing from the run entirely is the failure mode where an --elements typo, // or an unreachable loop, removes a whole entity type without anything going red. - expect(run.phases.map((p) => p.operation)).toEqual([...ALL_OPERATIONS]); - expect(run.ranPhases.map((p) => p.operation)).toEqual([...ALL_OPERATIONS]); + expect(run.phases.map((p) => p.operation)).toEqual([...ALL_OPERATIONS, LIST_ORDER]); + expect(run.ranPhases.map((p) => p.operation)).toEqual([...ALL_OPERATIONS, LIST_ORDER]); + }); + + it("runs list order after the push phases without failing", async () => { + harness = makeHarness(); + + const run = await harness.run(); + const listOrder = run.phases.find((p) => p.operation === LIST_ORDER)!; + + // One new item in a one-item list: the phase asks for the source order and nothing moves. + expect(listOrder.failed).toBe(0); + expect(harness.fake.callsTo("contentMethods.setContentListItemOrder")).toHaveLength(1); }); it("creates exactly one entity in each phase and fails none", async () => { @@ -70,7 +85,7 @@ describe("full instance — all eight phases", () => { const run = await harness.run(); - run.phases.forEach((phase) => { + pushPhases(run.phases).forEach((phase) => { expect({ op: phase.operation, ok: phase.successful, failed: phase.failed }).toEqual({ op: phase.operation, ok: 1, @@ -139,7 +154,7 @@ describe("full instance — all eight phases", () => { const run2 = await harness.resync(); - run2.phases.forEach((phase) => { + pushPhases(run2.phases).forEach((phase) => { expect({ op: phase.operation, ok: phase.successful, skip: phase.skipped, failed: phase.failed }).toEqual({ op: phase.operation, ok: 0, @@ -155,10 +170,21 @@ describe("full instance — all eight phases", () => { await harness.resync(); - const writes = harness.fake.calls.filter((c) => /save|axios\.post/i.test(c.method)); + const writes = harness.fake.calls.filter((c) => /save|axios\.post|setContentListItemOrder/i.test(c.method)); expect(writes.map((c) => c.method)).toEqual([]); }); + it("leaves list order alone: the target is already in the source's order", async () => { + harness = makeHarness(); + await harness.run(); + + const run2 = await harness.resync(); + + // resync() starts a fresh call log, so this is run 2's calls only. + expect(harness.fake.callsTo("contentMethods.setContentListItemOrder")).toHaveLength(0); + expect(run2.phases.find((p) => p.operation === LIST_ORDER)).toMatchObject({ successful: 0, failed: 0, skipped: 0 }); + }); + it("leaves the mapping tree identical to run 1", async () => { harness = makeHarness(); await harness.run(); @@ -216,7 +242,7 @@ describe("full instance — all eight phases", () => { const models = run2.phases.find((p) => p.operation === "pushModels")!; expect(models.successful).toBe(1); - run2.phases + pushPhases(run2.phases) .filter((p) => p.operation !== "pushModels") .forEach((phase) => { expect({ op: phase.operation, ok: phase.successful, skip: phase.skipped }).toEqual({ diff --git a/src/tests/sync/helpers/fake-instance.ts b/src/tests/sync/helpers/fake-instance.ts index 4c89476..9df33ea 100644 --- a/src/tests/sync/helpers/fake-instance.ts +++ b/src/tests/sync/helpers/fake-instance.ts @@ -218,6 +218,11 @@ export class FakeInstance { // ─── the mgmtApi.ApiClient surface ───────────────────────────────────────── modelMethods = { + deleteModel: async (id: number, guid: string) => { + this.record("modelMethods.deleteModel", guid, { id }); + this.instance(guid).models.delete(id); + return "Model deleted successfully."; + }, saveModel: async (model: any, guid: string) => { this.record("modelMethods.saveModel", guid, model); const store = this.instance(guid); @@ -243,6 +248,11 @@ export class FakeInstance { }; containerMethods = { + deleteContainer: async (id: number, guid: string) => { + this.record("containerMethods.deleteContainer", guid, { id }); + this.instance(guid).containers.delete(id); + return "Container deleted successfully."; + }, saveContainer: async (container: any, guid: string) => { this.record("containerMethods.saveContainer", guid, container); const store = this.instance(guid); @@ -275,6 +285,33 @@ export class FakeInstance { private lastPageSaveIds: number[] = []; contentMethods = { + /** Like the API: listed items first, the rest after in their order, renumbered 0..n-1 (skipUnknownIds semantics). */ + setContentListItemOrder: async (referenceName: string, guid: string, locale: string, contentIDs: number[], skipUnknownIds = false) => { + this.record("contentMethods.setContentListItemOrder", guid, { referenceName, contentIDs, skipUnknownIds }, locale); + const items = Array.from(this.instance(guid).content.entries()) + .filter(([key, item]) => key.startsWith(`${locale}:`) && `${item?.properties?.referenceName}`.toLowerCase() === referenceName.toLowerCase()) + .map(([, item]) => item) + .sort((a, b) => (a.properties?.itemOrder ?? Infinity) - (b.properties?.itemOrder ?? Infinity) || a.contentID - b.contentID); + const inList = new Set(items.map((i) => i.contentID)); + const skipped = contentIDs.filter((id) => !inList.has(id)); + if (skipped.length > 0 && !skipUnknownIds) throw new Error(`Content IDs not in ${referenceName}: ${skipped.join(", ")}`); + const listed = contentIDs.filter((id) => inList.has(id)); + const order = [...listed, ...items.map((i) => i.contentID).filter((id) => !listed.includes(id))]; + const updated: number[] = []; + order.forEach((id, index) => { + const item = items.find((i) => i.contentID === id)!; + if (item.properties?.itemOrder !== index) { + item.properties = { ...item.properties, itemOrder: index }; + updated.push(id); + } + }); + return { contentIDs: order, itemsUpdated: updated.length, updatedContentIDs: updated, skippedContentIDs: skipped }; + }, + deleteContent: async (id: number, guid: string, locale: string) => { + this.record("contentMethods.deleteContent", guid, { id }, locale); + this.instance(guid).content.delete(`${locale}:${id}`); + return [3000 + this.callsTo("contentMethods.deleteContent").length]; + }, saveContentItems: async (items: any[], guid: string, locale: string) => { this.record("contentMethods.saveContentItems", guid, items, locale); const store = this.instance(guid); @@ -309,6 +346,11 @@ export class FakeInstance { }; assetMethods = { + deleteGallery: async (guid: string, id: number) => { + this.record("assetMethods.deleteGallery", guid, { id }); + this.instance(guid).galleries.delete(id); + return "Gallery deleted successfully."; + }, getMediaList: async (pageSize: number, recordOffset: number, guid: string) => { this.record("assetMethods.getMediaList", guid, { pageSize, recordOffset }); // One page only: returning the full list on every call would loop forever if the @@ -350,6 +392,9 @@ export class FakeInstance { }, deleteFile: async (mediaID: number, guid: string) => { this.record("assetMethods.deleteFile", guid, { mediaID }); + const assets = this.instance(guid).assets; + const index = assets.findIndex((a: any) => a.mediaID === mediaID); + if (index >= 0) assets.splice(index, 1); return true; }, deleteFolder: async (originKey: string, guid: string) => { @@ -359,6 +404,16 @@ export class FakeInstance { }; pageMethods = { + deletePage: async (id: number, guid: string, locale: string) => { + this.record("pageMethods.deletePage", guid, { id }, locale); + this.instance(guid).pages.delete(`${locale}:${id}`); + return [4000 + this.callsTo("pageMethods.deletePage").length]; + }, + deletePageTemplate: async (guid: string, locale: string, id: number) => { + this.record("pageMethods.deletePageTemplate", guid, { id }, locale); + this.instance(guid).templates.delete(id); + return "Template deleted successfully."; + }, getSitemap: async (guid: string, locale: string) => { this.record("pageMethods.getSitemap", guid, undefined, locale); return this.instance(guid).sitemap; @@ -401,6 +456,15 @@ export class FakeInstance { }, }; + urlRedirectionMethods = { + deleteUrlRedirections: async (guid: string, ids: number[]) => { + this.record("urlRedirectionMethods.deleteUrlRedirections", guid, { ids }); + const store = this.instance(guid).urlRedirections; + const deleted = ids.filter((id) => store.delete(id)); + return { deleted, notFound: ids.filter((id) => !deleted.includes(id)) }; + }, + }; + instanceMethods = { getLocales: async (guid: string) => { this.record("instanceMethods.getLocales", guid); diff --git a/src/tests/sync/helpers/run-sync.ts b/src/tests/sync/helpers/run-sync.ts index 47ab159..0a07e9f 100644 --- a/src/tests/sync/helpers/run-sync.ts +++ b/src/tests/sync/helpers/run-sync.ts @@ -2,7 +2,7 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -import { state, resetState, setState } from "core/state"; +import { state, resetState, setState, enableReverseSync } from "core/state"; import { Pushers, PushResults } from "lib/pushers/orchestrate-pushers"; import type { JsonSummaryPhase } from "core/json-summary"; @@ -64,6 +64,8 @@ export interface RunSyncOptions { fixture: SyncFixture; /** Pre-seed the fake's target instance, for entities that exist server-side already. */ seedTarget?: (fake: FakeInstance, targetGuid: string) => void; + /** Extra CLI state for every run (e.g. { syncDeletes: true }). Can be changed between runs. */ + stateOverrides?: Record; } export interface SyncRun { @@ -84,6 +86,13 @@ export class SyncHarness { private readonly elements: string; private readonly overwrite: boolean; private readonly preflight: boolean; + /** Extra CLI state applied on every run; reassign between runs to change flags for run 2. */ + stateOverrides: Record; + /** + * Run as `reverse-sync`: same pair and mapping files, pushing target -> source. Set it between + * runs, after projecting the target to disk (it is the source of a reverse run). + */ + reverse = false; constructor(opts: RunSyncOptions) { this.sourceGuid = opts.sourceGuid ?? "src-1"; @@ -92,6 +101,7 @@ export class SyncHarness { this.elements = opts.elements ?? "Models,Containers,Content"; this.overwrite = opts.overwrite ?? false; this.preflight = opts.preflight ?? false; + this.stateOverrides = opts.stateOverrides ?? {}; this.rootPath = fs.mkdtempSync(path.join(os.tmpdir(), "agility-sync-harness-")); this.fake = new FakeInstance({ locales: this.locales }); @@ -130,7 +140,9 @@ export class SyncHarness { elements: this.elements, overwrite: this.overwrite, preflight: this.preflight, + ...this.stateOverrides, }); + if (this.reverse) enableReverseSync(); state.cachedApiClient = this.fake.asApiClient(); state.availableLocales = [...this.locales]; this.wireBatchLayer(); diff --git a/src/tests/sync/sync-deletes.test.ts b/src/tests/sync/sync-deletes.test.ts new file mode 100644 index 0000000..116ceb4 --- /dev/null +++ b/src/tests/sync/sync-deletes.test.ts @@ -0,0 +1,248 @@ +/** + * --syncDeletes end to end: sync, delete something in the source, re-sync with the flag, and the + * target loses exactly that — through the real orchestrator, mappings and all. + */ +import * as fs from "fs"; +import * as path from "path"; +import { SyncHarness } from "./helpers/run-sync"; +import { installNetworkStubs, InstalledNetworkStubs } from "./helpers/network-stubs"; + +jest.mock("lib/pushers/batch-polling", () => ({ + pollBatchUntilComplete: jest.fn(), + extractContentBatchResults: jest.fn(), + extractPageBatchResults: jest.fn(), +})); + +// Redirection saves build their own URL and call global fetch; the harness wires this to the fake. +// Deletes go through the SDK client (urlRedirectionMethods), which the fake implements directly. +jest.mock("lib/pushers/url-redirection-api", () => ({ + MAX_URL_REDIRECTION_BATCH_SIZE: 250, + saveUrlRedirections: jest.fn(), +})); + +const POST_MODEL = { + id: 10, + displayName: "Post", + referenceName: "Post", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Title", label: "Title", required: true }], +}; + +const AUTHOR_MODEL = { + id: 11, + displayName: "Author", + referenceName: "Author", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Name", label: "Name", required: true }], +}; + +const NEWS_MODEL = { + id: 12, + displayName: "News", + referenceName: "News", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Headline", label: "Headline", required: true }], +}; + +const POSTS_CONTAINER = { + contentViewID: 200, + referenceName: "Posts", + title: "Posts", + contentDefinitionID: 10, + contentDefinitionName: "Post", + lastModifiedDate: "01/01/2025 12:00AM", + isShared: false, +}; + +const AUTHORS_CONTAINER = { + contentViewID: 201, + referenceName: "Authors", + title: "Authors", + contentDefinitionID: 11, + contentDefinitionName: "Author", + lastModifiedDate: "01/01/2025 12:00AM", + isShared: false, +}; + +function fixture() { + return { + source: { + "models/10.json": POST_MODEL, + "models/11.json": AUTHOR_MODEL, + "models/12.json": NEWS_MODEL, + "containers/200.json": POSTS_CONTAINER, + "containers/201.json": AUTHORS_CONTAINER, + }, + }; +} + +describe("--syncDeletes through the orchestrator", () => { + let harness: SyncHarness; + let net: InstalledNetworkStubs; + + beforeEach(() => { + net = installNetworkStubs(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + harness?.cleanup(); + net.restore(); + jest.restoreAllMocks(); + }); + + function deleteFromSource(rel: string) { + fs.rmSync(path.join(harness.rootPath, harness.sourceGuid, rel)); + } + + it("deletes the target counterparts of source deletes and drops their mappings", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const authorTarget = harness.readMappings("models")!.find((m) => m.sourceID === 11)!.targetID; + const authorsContainerTarget = harness.readMappings("containers")!.find((m) => m.sourceContentViewID === 201)!.targetContentViewID; + + deleteFromSource("models/11.json"); + deleteFromSource("containers/201.json"); + harness.stateOverrides = { syncDeletes: true }; + const run2 = await harness.resync(); + + const target = harness.fake.instance(harness.targetGuid); + expect(target.models.has(authorTarget)).toBe(false); + expect(target.containers.has(authorsContainerTarget)).toBe(false); + expect(target.models.size).toBe(2); + // Container before its model. + const order = harness.fake.calls.map((c) => c.method); + expect(order.indexOf("containerMethods.deleteContainer")).toBeLessThan(order.indexOf("modelMethods.deleteModel")); + + expect(harness.readMappings("models")!.map((m) => m.sourceID).sort()).toEqual([10, 12]); + expect(harness.readMappings("containers")!.map((m) => m.sourceContentViewID)).toEqual([200]); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toMatchObject({ successful: 2, failed: 0 }); + }); + + it("deletes on reverse-sync too: what was deleted on the target goes from the source", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + // A reverse run pushes into the original source, so give the fake the source's entities. + [POST_MODEL, AUTHOR_MODEL, NEWS_MODEL].forEach((m) => harness.fake.seedModel(harness.sourceGuid, { ...m })); + [POSTS_CONTAINER, AUTHORS_CONTAINER].forEach((c) => harness.fake.seedContainer(harness.sourceGuid, { ...c })); + + // Delete Author and its container on the target, then pull the target (the reverse run's source). + const target = harness.fake.instance(harness.targetGuid); + const authorTarget = harness.readMappings("models")!.find((m) => m.sourceID === 11)!.targetID; + const authorsContainerTarget = harness.readMappings("containers")!.find((m) => m.sourceContentViewID === 201)!.targetContentViewID; + target.containers.delete(authorsContainerTarget); + target.models.delete(authorTarget); + fs.rmSync(path.join(harness.rootPath, harness.targetGuid), { recursive: true, force: true }); + harness.fake.projectToDisk(harness.rootPath, harness.targetGuid); + harness.fake.resetCalls(); + + harness.reverse = true; + harness.stateOverrides = { syncDeletes: true }; + const run2 = await harness.run(); + + const source = harness.fake.instance(harness.sourceGuid); + expect(source.models.has(11)).toBe(false); + expect(source.containers.has(201)).toBe(false); + expect(Array.from(source.models.keys()).sort()).toEqual([10, 12]); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toMatchObject({ successful: 2, failed: 0 }); + // The mapping files keep the forward orientation (source = A) and lose the pair. + expect(harness.readMappings("models")!.map((m) => m.sourceID).sort()).toEqual([10, 12]); + expect(harness.readMappings("containers")!.map((m) => m.sourceContentViewID)).toEqual([200]); + }); + + it("leaves source deletes alone without the flag", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + + deleteFromSource("models/12.json"); + const run2 = await harness.resync(); + + expect(harness.fake.instance(harness.targetGuid).models.size).toBe(3); + expect(harness.fake.callsTo("modelMethods.deleteModel")).toEqual([]); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toBeUndefined(); + }); + + it("keeps a model that a target-only container still uses", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const newsTarget = harness.readMappings("models")!.find((m) => m.sourceID === 12)!.targetID; + // Someone built a list on the target from the News model. + harness.fake.seedContainer(harness.targetGuid, { + contentViewID: 8999, + referenceName: "TargetNews", + contentDefinitionID: newsTarget, + lastModifiedDate: "01/01/2025 12:00AM", + }); + + deleteFromSource("models/12.json"); + harness.stateOverrides = { syncDeletes: true }; + const run2 = await harness.resync(); + + expect(harness.fake.instance(harness.targetGuid).models.has(newsTarget)).toBe(true); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toMatchObject({ successful: 0, skipped: 1 }); + const warning = run2.results[0].warningDetails.find((w: any) => w.name === "News"); + expect(warning?.error).toContain("container TargetNews"); + }); + + it("previews deletes under --preflight without writing", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const before = harness.readAllMappings(); + + deleteFromSource("models/12.json"); + harness.stateOverrides = { syncDeletes: true, preflight: true }; + await harness.resync(); + + expect(harness.fake.callsTo("modelMethods.deleteModel")).toEqual([]); + expect(harness.readAllMappings()).toEqual(before); + }); + + it("deletes URL redirections through the SDK's urlRedirectionMethods", async () => { + const redirect = (id: number, origin: string) => ({ id, originUrl: origin, destinationUrl: "/new", statusCode: 301 }); + harness = new SyncHarness({ + elements: "UrlRedirections", + fixture: { + source: { + "en-us/urlredirections/urlredirections.json": { + items: [redirect(1, "/keep"), redirect(2, "/gone")], + lastAccessDate: "2025-01-01T00:00:00.000", + }, + }, + }, + }); + await harness.run(); + const goneTarget = harness.readMappings("urlredirections")!.find((m) => m.sourceUrlRedirectionID === 2)!.targetUrlRedirectionID; + + fs.writeFileSync( + path.join(harness.rootPath, harness.sourceGuid, "en-us", "urlredirections", "urlredirections.json"), + JSON.stringify({ items: [redirect(1, "/keep")], lastAccessDate: "2025-01-02T00:00:00.000" }) + ); + harness.stateOverrides = { syncDeletes: true }; + await harness.resync(); + + expect(harness.fake.callsTo("urlRedirectionMethods.deleteUrlRedirections")).toEqual([ + expect.objectContaining({ guid: harness.targetGuid, payload: { ids: [goneTarget] } }), + ]); + expect(harness.fake.instance(harness.targetGuid).urlRedirections.has(goneTarget)).toBe(false); + expect(harness.readMappings("urlredirections")!.map((m) => m.sourceUrlRedirectionID)).toEqual([1]); + expect(net.fetchCalls).toEqual([]); + }); + + it("refuses --syncDeletes on a scoped run before writing anything", async () => { + harness = new SyncHarness({ + elements: "Models,Containers", + fixture: fixture(), + stateOverrides: { syncDeletes: true, models: "Post" }, + }); + + const run = await harness.run(); + + expect(run.results[0].failed[0].error).toContain("Sync deletes validation failed"); + expect(harness.fake.calls.filter((c) => !c.method.startsWith("get"))).toEqual([]); + }); +}); diff --git a/yarn.lock b/yarn.lock index 2f089c6..f0748a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,10 +19,10 @@ dotenv "^8.2.0" proper-lockfile "^4.1.2" -"@agility/management-sdk@^0.1.38": - version "0.1.38" - resolved "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.38.tgz" - integrity sha512-g6/hNgCjf+uzcJbkPaxeWqwaAid50tMnRW0KYQ4J/fGuJqtcZG0et0X3m76Ev3yUqnkpIO/+C5zMIQMghN3/oQ== +"@agility/management-sdk@^0.2.0-beta.3": + version "0.2.0-beta.3" + resolved "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.2.0-beta.3.tgz" + integrity sha512-jBe7eJxOS7HdhfEnuCBBWwPZIGuA4qR+ekwU+wKmMk8HgMVc33BhuQUSajvwSjaZFkrVhTozjYiHNQjz2Y4GVg== dependencies: axios "^0.27.2" @@ -3251,7 +3251,7 @@ rx-lite@*, rx-lite@^4.0.8: resolved "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz" integrity sha512-Cun9QucwK6MIrp3mry/Y7hqD1oFqTYLQ4pGxaHTjIdaFDWRGGLikqp6u8LcWJnzpoALg9hap+JGk8sFIUuEGNA== -rxjs@^7.5.5, rxjs@^7.8.2: +rxjs@^7.8.2: version "7.8.2" resolved "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz" integrity sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==