From effb165b3cf448f6ccda7e9f506caf811f2e9c95 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 30 Oct 2026 12:07:22 -0400 Subject: [PATCH 1/3] Add --syncDeletes: delete on the target what was deleted in the source A final sync phase, opt-in per run, that deletes the target counterpart of every mapped entity whose source is gone: URL redirections, pages (children first), content, containers, templates, models, assets and galleries, in dependency order. Only entities reached through a mapping are considered. Safety rules: - Content and pages are confirmed deleted against the source API (404 or state 3); a full pull also omits unpublished items, which are kept. - An empty source list for a type is never read as "everything deleted". - A target edited since the last sync is a conflict unless --overwrite. - A target that something staying on the target still uses (child pages, linking content or pages, container items, model usage, embedded asset URLs, gallery assets) is kept and reported. - Mass-delete guard: more than 50% and more than 10 of one type deletes nothing and fails unless --allowMassDeletes. - Refused with --pages, --containers, --models, --models-with-deps; not primed from .env. Deleted and already-gone targets drop their mapping records. --preflight reports planned deletes with a new "delete" action; --jsonSummary gets a syncDeletes phase and option. URL redirections are deleted through the Management API's bulk DELETE endpoint, called directly until the CLI moves to an SDK release with urlRedirectionMethods.deleteUrlRedirections. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 + src/core/json-summary.ts | 10 +- src/core/push.ts | 1 + src/core/state.ts | 8 + src/core/system-args.ts | 18 + src/core/tests/json-summary.test.ts | 8 +- src/core/tests/push.test.ts | 2 +- src/lib/deleters/delete-plan.ts | 334 ++++++ src/lib/deleters/sync-deletes.ts | 988 ++++++++++++++++++ src/lib/deleters/tests/delete-plan.test.ts | 177 ++++ src/lib/deleters/tests/sync-deletes.test.ts | 358 +++++++ src/lib/preflight/preflight-report.ts | 19 +- .../preflight/tests/preflight-report.test.ts | 36 +- src/lib/pushers/orchestrate-pushers.ts | 31 + src/lib/pushers/url-redirection-api.ts | 51 +- src/tests/sync/helpers/fake-instance.ts | 33 + src/tests/sync/helpers/run-sync.ts | 6 + src/tests/sync/sync-deletes.test.ts | 179 ++++ src/types/urlRedirection.ts | 8 + 19 files changed, 2265 insertions(+), 22 deletions(-) create mode 100644 src/lib/deleters/delete-plan.ts create mode 100644 src/lib/deleters/sync-deletes.ts create mode 100644 src/lib/deleters/tests/delete-plan.test.ts create mode 100644 src/lib/deleters/tests/sync-deletes.test.ts create mode 100644 src/tests/sync/sync-deletes.test.ts diff --git a/README.md b/README.md index ab351a0..7938ba7 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,20 @@ agility pull [options] | `--elements` | string | `Models,Galleries,Assets,Containers,Content,Templates,Pages,Sitemaps,UrlRedirections` | Comma-separated list of elements to process | | `--models` | string | _(empty)_ | Comma-separated list of model reference names to sync (only syncs the specified models) | +#### Syncing deletes + +`--syncDeletes` adds a final phase to a sync. For every mapping whose source entity is gone from the source pull it decides: + +| Outcome | When | +| --- | --- | +| **deleted** | The source entity is deleted and the target is unchanged since the last sync (or `--overwrite` is set). The mapping is removed. | +| **conflict** (kept) | The target was edited since the last sync. Re-run with `--overwrite` to delete it anyway. | +| **blocked** (kept) | Something that stays on the target still uses it — e.g. a target-only child page, a page or item that links to the content item, items in the container, a container or item built on the model, content that embeds the asset. | +| **mapping removed** | The target is already gone, or another live source entity maps to the same target. | +| **kept** | The source item is unpublished rather than deleted, or the delete could not be confirmed. | + +Run it with `--preflight` first: the report lists every planned delete under `Delete …` phases. Deleted pages and content items go to the instance's normal delete flow (the batch is labelled "Deleted by agility-cli --syncDeletes"). + **Authentication & Environment Options:** | Option | Type | Default | Description | @@ -105,6 +119,8 @@ agility sync [options] | Option | Type | Default | Description | | --------------- | ------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--overwrite` | boolean | `false` | Conflict-scoped override. By default, when a target item has its own changes that conflict with the source, the CLI skips it to avoid data loss. With `--overwrite`, those conflicting target items are overwritten with the source version. Non-conflicting updates are applied either way. Also covers a mapped target item that no longer exists (deleted or unpublished on the target): by default that is reported as a conflict and skipped, with `--overwrite` the item is recreated and the mapping repointed. | +| `--syncDeletes` | boolean | `false` | After syncing, delete from the target the entities that were deleted in the source since the last sync (URL redirections, pages, content, containers, templates, models, assets, galleries — dependents first). Only entities the CLI previously synced (that have a mapping) are considered; nothing created directly on the target is touched. A target entity edited since the last sync is reported as a conflict and kept unless `--overwrite` is also set. An entity that something kept on the target still uses (a child page, a page or item that links to the content, a container's items, ...) is kept and reported. Content and page deletes are confirmed against the source first, so an item that was only unpublished is never deleted. Cannot be combined with `--pages`, `--containers`, `--models` or `--models-with-deps`; use `--elements` to limit the types. Works with `--preflight`. Not read from `.env`. See [Syncing deletes](#syncing-deletes). | +| `--allowMassDeletes` | boolean | `false` | With `--syncDeletes`: allow a run to delete more than half (and more than 10) of the mapped entities of one type. Without it such a run deletes nothing and fails, because a stale or partial local cache looks the same as a mass delete. | | `--autoPublish` | string | _(disabled)_ | Automatically publish synced items that were published in the source instance. Values: `content`, `pages`, `both`. If flag is provided without a value, defaults to `both`. Items that are only in staging (not published) in the source are skipped. | | `--fullPull` | boolean | `false` | Full content re-pull. Discards the stored content sync token for every locale, re-pulls all content items and pages, and removes local content/page files that no longer exist on the instance. Use when a local cache is suspected stale. Models, containers, templates, galleries and assets reconcile against the instance on every pull regardless. | | `--preflight` | boolean | `false` | Dry-run preview. Runs the full source-pull, target-pull, dependency analysis and change detection, then reports the creates/updates/skips/conflicts a real sync would produce — **without writing anything** to the target instance or mapping files. Exits non-zero when conflicts are detected, so CI can gate a real sync on a clean preflight. | @@ -141,6 +157,10 @@ agility sync --sourceGuid="abc123" --targetGuid="def456" --overwrite # Preview what a sync would do without writing anything (exits non-zero if conflicts are found) agility sync --sourceGuid="abc123" --targetGuid="def456" --preflight +# Also delete from the target what was deleted in the source since the last sync (preview first) +agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes --preflight +agility sync --sourceGuid="abc123" --targetGuid="def456" --syncDeletes + # Sync only specified models (models only, no dependencies) agility sync --sourceGuid="abc123" --targetGuid="def456" --models="BlogPost,BlogCategory" diff --git a/src/core/json-summary.ts b/src/core/json-summary.ts index 9e62f3f..c421a8c 100644 --- a/src/core/json-summary.ts +++ b/src/core/json-summary.ts @@ -88,6 +88,8 @@ export interface JsonSummaryPreflightPhase { phase: string; create: number; update: number; + /** --syncDeletes: target entities a real run would delete. */ + delete: number; skip: number; conflict: number; /** PROD-2542: would-be writes that fail validation against the target model. */ @@ -95,13 +97,13 @@ export interface JsonSummaryPreflightPhase { } export interface JsonSummaryPreflight { - totals: { create: number; update: number; skip: number; conflict: number; invalid: number }; + totals: { create: number; update: number; delete: number; skip: number; conflict: number; invalid: number }; hasConflicts: boolean; phases: JsonSummaryPreflightPhase[]; /** Every recorded action, in the order the pushers recorded them. */ entries: Array<{ phase: string; - action: "create" | "update" | "skip" | "conflict" | "invalid"; + action: "create" | "update" | "delete" | "skip" | "conflict" | "invalid"; name: string; locale?: string; detail?: string; @@ -123,6 +125,8 @@ export interface JsonSummary { options: { preflight: boolean; overwrite: boolean; + /** --syncDeletes. Optional so summaries written before the flag existed still parse. */ + syncDeletes?: boolean; autoPublish: string; elements: string[]; locales: string[]; @@ -235,6 +239,7 @@ export function emitAbortSummary( targetGuid?: string; preflight?: boolean; overwrite?: boolean; + syncDeletes?: boolean; autoPublish?: string; elements?: string; locale?: string[]; @@ -253,6 +258,7 @@ export function emitAbortSummary( options: { preflight: runState.preflight === true, overwrite: runState.overwrite === true, + syncDeletes: runState.syncDeletes === true, autoPublish: runState.autoPublish || "", elements: (runState.elements || "").split(",").filter(Boolean), locales: Array.isArray(runState.locale) ? runState.locale : [], diff --git a/src/core/push.ts b/src/core/push.ts index 038b0ce..37fc063 100644 --- a/src/core/push.ts +++ b/src/core/push.ts @@ -490,6 +490,7 @@ export class Push { options: { preflight: state.preflight === true, overwrite: state.overwrite === true, + syncDeletes: state.syncDeletes === true, autoPublish: state.autoPublish || "", elements: (state.elements || "").split(",").filter(Boolean), locales: Array.isArray(state.locale) ? state.locale : [], diff --git a/src/core/state.ts b/src/core/state.ts index 0305535..1500ab1 100644 --- a/src/core/state.ts +++ b/src/core/state.ts @@ -36,6 +36,8 @@ export interface State { // Operation control overwrite: boolean; + syncDeletes: boolean; // Delete from the target what was deleted in the source since the last sync + allowMassDeletes: boolean; // Lift the --syncDeletes mass-delete guard preflight: boolean; // Preflight mode - report planned sync actions without writing to target/mappings fullPull: boolean; // PROD-2614: discard content sync tokens and do a full content re-pull that removes stale local files jsonSummary: string; // Path to write the machine-readable run summary to; empty = disabled @@ -166,6 +168,8 @@ export const state: State = { // Operation control overwrite: false, + syncDeletes: false, + allowMassDeletes: false, preflight: false, fullPull: false, jsonSummary: "", @@ -276,6 +280,8 @@ export function setState(argv: any) { // Operation control if (argv.overwrite !== undefined) state.overwrite = argv.overwrite; + if (argv.syncDeletes !== undefined) state.syncDeletes = argv.syncDeletes; + if (argv.allowMassDeletes !== undefined) state.allowMassDeletes = argv.allowMassDeletes; // Operation control (preflight) if (argv.preflight !== undefined) state.preflight = argv.preflight; @@ -473,6 +479,8 @@ export function resetState() { // Operation control state.overwrite = false; + state.syncDeletes = false; + state.allowMassDeletes = false; state.preflight = false; state.fullPull = false; state.jsonSummary = ""; diff --git a/src/core/system-args.ts b/src/core/system-args.ts index 69aba1a..69f6cc1 100644 --- a/src/core/system-args.ts +++ b/src/core/system-args.ts @@ -154,6 +154,22 @@ export const systemArgs = { default: false, }, + // Sync deletes. Deliberately not primed from .env: a destructive mode is opted into per run. + syncDeletes: { + describe: + "(sync/push only) After syncing, delete from the target the entities that were deleted in the source since the last sync. Only entities the CLI previously synced (that have a mapping) are considered — nothing created directly on the target is touched. A target entity that was changed since the last sync is a conflict and is kept unless --overwrite is also set; one that something else on the target still uses is kept and reported. Content and page deletes are confirmed against the source first (an unpublished item is not deleted). Default: false.", + type: "boolean" as const, + alias: ["sync-deletes", "syncdeletes", "SyncDeletes", "SYNC_DELETES", "SYNCDELETES"], + default: false, + }, + allowMassDeletes: { + describe: + "(with --syncDeletes) Allow a sync to delete more than half of the mapped entities of a kind. Without it such a run refuses to delete anything, since a stale or partial local cache looks the same as a mass delete. Default: false.", + type: "boolean" as const, + alias: ["allow-mass-deletes", "allowmassdeletes", "AllowMassDeletes", "ALLOW_MASS_DELETES"], + default: false, + }, + // Auto-publish after sync autoPublish: { describe: @@ -267,6 +283,8 @@ export interface SystemArgs { preflight?: boolean; // Preflight mode - report planned sync actions without writing to target/mappings verbose?: boolean; overwrite?: boolean; + syncDeletes?: boolean; + allowMassDeletes?: boolean; elements?: string; guid?: string; sourceGuid?: string; diff --git a/src/core/tests/json-summary.test.ts b/src/core/tests/json-summary.test.ts index c43f391..7dd395d 100644 --- a/src/core/tests/json-summary.test.ts +++ b/src/core/tests/json-summary.test.ts @@ -181,9 +181,9 @@ describe("buildJsonSummary — shape", () => { const summary = buildJsonSummary( makeInput({ preflight: { - totals: { create: 2, update: 1, skip: 0, conflict: 1, invalid: 0 }, + totals: { create: 2, update: 1, delete: 0, skip: 0, conflict: 1, invalid: 0 }, hasConflicts: true, - phases: [{ phase: "Models", create: 2, update: 1, skip: 0, conflict: 1, invalid: 0 }], + phases: [{ phase: "Models", create: 2, update: 1, delete: 0, skip: 0, conflict: 1, invalid: 0 }], entries: [{ phase: "Models", action: "conflict", name: "LinkCard" }], }, }) @@ -395,9 +395,9 @@ describe("buildJsonSummary — required-field validation failures (PROD-2542)", it("carries the preflight invalid count additively", () => { const preflight = { - totals: { create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }, + totals: { create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }, hasConflicts: false, - phases: [{ phase: "Content", create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }], + phases: [{ phase: "Content", create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }], entries: [ { phase: "Content", action: "invalid" as const, name: "home_promobanner", locale: "en-us", detail: "x" }, { phase: "Content", action: "create" as const, name: "ok", locale: "en-us" }, diff --git a/src/core/tests/push.test.ts b/src/core/tests/push.test.ts index a4526a6..3bdc1aa 100644 --- a/src/core/tests/push.test.ts +++ b/src/core/tests/push.test.ts @@ -410,7 +410,7 @@ describe("Push.pushInstances — --jsonSummary", () => { const parsed = JSON.parse(fs.readFileSync(out, "utf8")); expect(parsed.exitCode).toBe(0); expect(process.exitCode).toBeUndefined(); - expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(parsed.preflight.totals).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); expect(parsed.preflight.hasConflicts).toBe(false); expect(parsed.warnings.some((w: any) => w.code === "foreignCdnReference")).toBe(true); diff --git a/src/lib/deleters/delete-plan.ts b/src/lib/deleters/delete-plan.ts new file mode 100644 index 0000000..1c2903e --- /dev/null +++ b/src/lib/deleters/delete-plan.ts @@ -0,0 +1,334 @@ +/** + * --syncDeletes: decide what happens to each target entity whose source counterpart is gone. + * + * A candidate is a mapping record whose source entity is no longer in the source pull. Whether it + * is actually deleted on the target is decided here from facts the caller gathers (is the source + * really deleted, is the target still there, did someone edit it since the last sync, does + * anything that stays depend on it). Everything in this file is pure so the rules can be tested + * without an instance; sync-deletes.ts does the reading and the API calls. + */ + +export type DeleteKind = + | "urlRedirection" + | "page" + | "content" + | "container" + | "template" + | "model" + | "asset" + | "gallery"; + +/** Execution order: dependents before the things they depend on. */ +export const DELETE_ORDER: DeleteKind[] = [ + "urlRedirection", + "page", + "content", + "container", + "template", + "model", + "asset", + "gallery", +]; + +/** --elements name for each kind. A kind is only considered when its element is in the run. */ +export const DELETE_KIND_ELEMENT: Record = { + urlRedirection: "UrlRedirections", + page: "Pages", + content: "Content", + container: "Containers", + template: "Templates", + model: "Models", + asset: "Assets", + gallery: "Galleries", +}; + +/** Mapping folder for each kind (content and pages are per-locale). */ +export const DELETE_KIND_MAPPING_TYPE: Record = { + urlRedirection: "urlredirections", + page: "page", + content: "item", + container: "containers", + template: "templates", + model: "models", + asset: "assets", + gallery: "galleries", +}; + +/** Source/target ID fields of each kind's mapping records. */ +export const DELETE_KIND_ID_FIELDS: Record = { + urlRedirection: { source: "sourceUrlRedirectionID", target: "targetUrlRedirectionID" }, + page: { source: "sourcePageID", target: "targetPageID" }, + content: { source: "sourceContentID", target: "targetContentID" }, + container: { source: "sourceContentViewID", target: "targetContentViewID" }, + template: { source: "sourcePageTemplateID", target: "targetPageTemplateID" }, + model: { source: "sourceID", target: "targetID" }, + asset: { source: "sourceMediaID", target: "targetMediaID" }, + gallery: { source: "sourceMediaGroupingID", target: "targetMediaGroupingID" }, +}; + +export type DeleteAction = + /** Delete the target entity and drop its mapping. */ + | "delete" + /** The target was edited since the last sync; needs --overwrite. */ + | "conflict" + /** Something that is not being deleted still depends on the target entity. */ + | "blocked" + /** Nothing to delete (target already gone, or still owned by a live source); drop the mapping. */ + | "unmap" + /** The source is not confirmed deleted (e.g. only unpublished); leave everything as it is. */ + | "keep"; + +export interface DeleteCandidate { + kind: DeleteKind; + locale?: string; + sourceID: number; + targetID: number; + name: string; + /** The mapping record this candidate came from, removed again on delete/unmap. */ + record: any; + action: DeleteAction; + reason: string; + /** Pages: depth in the target sitemap, so children are deleted before their parents. */ + depth?: number; +} + +/** What the caller found out about one candidate. */ +export interface CandidateFacts { + /** true = confirmed deleted; a string = not confirmed, with the reason. */ + sourceDeleted: true | string; + targetExists: boolean; + targetChanged: boolean; + /** A live source entity that maps onto the same target. */ + sharedWithSourceID?: number; + /** Why the target cannot be deleted (what still depends on it). */ + blockedBy?: string; +} + +export function decideAction(facts: CandidateFacts, overwrite: boolean): { action: DeleteAction; reason: string } { + if (facts.sourceDeleted !== true) { + return { action: "keep", reason: facts.sourceDeleted }; + } + if (facts.sharedWithSourceID !== undefined) { + return { + action: "unmap", + reason: `target is also mapped from live source ID ${facts.sharedWithSourceID}; removing the stale mapping only`, + }; + } + if (!facts.targetExists) { + return { action: "unmap", reason: "already gone on the target; removing the mapping" }; + } + if (facts.blockedBy) { + return { action: "blocked", reason: facts.blockedBy }; + } + if (facts.targetChanged && !overwrite) { + return { + action: "conflict", + reason: "deleted in the source, but the target was changed since the last sync; use --overwrite to delete it", + }; + } + return { + action: "delete", + reason: facts.targetChanged + ? "deleted in the source (target changes discarded by --overwrite)" + : "deleted in the source", + }; +} + +export interface MappingPair { + sourceID: number; + targetID: number; + record: any; +} + +/** Usable (sourceID, targetID) pairs from a kind's mapping records. */ +export function mappingPairs(kind: DeleteKind, records: any[]): MappingPair[] { + const fields = DELETE_KIND_ID_FIELDS[kind]; + const out: MappingPair[] = []; + (records || []).forEach((record) => { + const sourceID = toId(record?.[fields.source]); + const targetID = toId(record?.[fields.target]); + if (sourceID === undefined || targetID === undefined) return; + out.push({ sourceID, targetID, record }); + }); + return out; +} + +/** + * Split a kind's mappings into the records whose source is gone (candidates) and the set of + * target IDs still owned by a live source. A target owned by a live source is never deleted. + */ +export function findCandidates( + pairs: MappingPair[], + liveSourceIDs: Set +): { candidates: MappingPair[]; liveTargets: Map } { + const candidates: MappingPair[] = []; + const liveTargets = new Map(); + pairs.forEach((pair) => { + if (liveSourceIDs.has(pair.sourceID)) { + if (!liveTargets.has(pair.targetID)) liveTargets.set(pair.targetID, pair.sourceID); + } else { + candidates.push(pair); + } + }); + return { candidates, liveTargets }; +} + +/** + * Mass-delete guard. A cache that was wiped, copied from another instance or only partly pulled + * looks exactly like a source that deleted everything. Refuse when a single kind (per locale) + * would lose more than `maxRatio` of its mapped entities and more than `minCount` of them. + */ +export const MASS_DELETE_MIN_COUNT = 10; +export const MASS_DELETE_MAX_RATIO = 0.5; + +export interface MassDeleteViolation { + kind: DeleteKind; + locale?: string; + deletes: number; + mapped: number; +} + +export function findMassDeletes( + groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }>, + minCount = MASS_DELETE_MIN_COUNT, + maxRatio = MASS_DELETE_MAX_RATIO +): MassDeleteViolation[] { + const out: MassDeleteViolation[] = []; + groups.forEach(({ kind, locale, mapped, candidates }) => { + const deletes = candidates.filter((c) => c.action === "delete").length; + if (deletes > minCount && mapped > 0 && deletes / mapped > maxRatio) { + out.push({ kind, locale, deletes, mapped }); + } + }); + return out; +} + +// ─── reference scanning ───────────────────────────────────────────────────── + +export interface FieldReferences { + contentIDs: Set; + /** Lower-cased container reference names (linked lists). */ + containers: Set; + galleryIDs: Set; +} + +/** + * Every content item, container and gallery a content item's fields point at, in any of the + * shapes the sync reads ({contentid}, {sortids}, {referencename}, {galleryid}, and the scalar + * "123" / "1,2,3" form a dropdown or checkbox list value field stores). Over-reporting only ever + * blocks a delete, so the scalar form is matched loosely. + */ +export function collectFieldReferences(fields: any): FieldReferences { + const refs: FieldReferences = { contentIDs: new Set(), containers: new Set(), galleryIDs: new Set() }; + const visit = (value: any) => { + if (value === null || value === undefined) return; + if (typeof value === "string") { + if (/^\s*\d+(\s*,\s*\d+)*\s*$/.test(value)) parseIdList(value).forEach((id) => refs.contentIDs.add(id)); + return; + } + if (Array.isArray(value)) { + value.forEach(visit); + return; + } + if (typeof value !== "object") return; + Object.keys(value).forEach((key) => { + const lowered = key.toLowerCase(); + const v = value[key]; + if (lowered === "contentid" || lowered === "sortids") { + parseIdList(v).forEach((id) => refs.contentIDs.add(id)); + } else if (lowered === "referencename" && typeof v === "string" && v) { + refs.containers.add(v.toLowerCase()); + } else if (lowered === "galleryid") { + parseIdList(v).forEach((id) => refs.galleryIDs.add(id)); + } else { + visit(v); + } + }); + }; + visit(fields); + return refs; +} + +export function parseIdList(value: any): number[] { + if (typeof value === "number") return value > 0 ? [value] : []; + if (typeof value !== "string") return []; + return value + .split(",") + .map((s) => parseInt(s.trim(), 10)) + .filter((n) => Number.isFinite(n) && n > 0); +} + +export function toId(value: any): number | undefined { + const n = typeof value === "string" ? parseInt(value, 10) : value; + return typeof n === "number" && Number.isFinite(n) && n > 0 ? n : undefined; +} + +/** Content IDs placed as modules in a page's zones. */ +export function pageZoneContentIDs(page: any): number[] { + const out: number[] = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (!Array.isArray(modules)) return; + modules.forEach((module) => { + const item = module?.item; + const id = toId(item?.contentid ?? item?.contentID ?? item?.ContentID); + if (id) out.push(id); + }); + }); + return out; +} + +/** Lower-cased model reference names of the modules placed in a page's zones. */ +export function pageZoneModuleNames(page: any): string[] { + const out: string[] = []; + const zones = page?.zones; + if (!zones || typeof zones !== "object") return out; + Object.keys(zones).forEach((zone) => { + const modules = zones[zone]; + if (!Array.isArray(modules)) return; + modules.forEach((module) => { + if (typeof module?.module === "string" && module.module) out.push(module.module.toLowerCase()); + }); + }); + return out; +} + +/** Lower-cased model reference names a model's fields link to (Content / linked-list fields). */ +export function modelFieldModelReferences(model: any): string[] { + const out: string[] = []; + (model?.fields || []).forEach((field: any) => { + const settings = field?.settings || {}; + Object.keys(settings).forEach((key) => { + if (key.toLowerCase() !== "contentdefinition") return; + const v = settings[key]; + if (typeof v === "string" && v) out.push(v.toLowerCase()); + }); + }); + return out; +} + +/** "a, b and 3 more" for blocker messages. */ +export function describeList(names: string[], max = 3): string { + const unique = Array.from(new Set(names)); + if (unique.length <= max) return unique.join(", "); + return `${unique.slice(0, max).join(", ")} and ${unique.length - max} more`; +} + +/** Flatten a sitemap's page tree into parent links and depths. */ +export function indexSitemap(channels: any[]): { parentOf: Map; depthOf: Map } { + const parentOf = new Map(); + const depthOf = new Map(); + const walk = (nodes: any[], parent: number | null, depth: number) => { + (nodes || []).forEach((node) => { + const id = toId(node?.pageID); + if (id === undefined) return; + parentOf.set(id, parent); + depthOf.set(id, depth); + walk(node?.childPages || [], id, depth + 1); + }); + }; + (channels || []).forEach((channel) => walk(channel?.pages || [], null, 0)); + return { parentOf, depthOf }; +} diff --git a/src/lib/deleters/sync-deletes.ts b/src/lib/deleters/sync-deletes.ts new file mode 100644 index 0000000..3dfcc00 --- /dev/null +++ b/src/lib/deleters/sync-deletes.ts @@ -0,0 +1,988 @@ +/** + * --syncDeletes: after every pusher has run, delete from the target what was deleted in the + * source since the last sync. + * + * Only entities reached through a mapping record are ever considered — something created on the + * target is never touched. For each mapping whose source entity is gone from the source pull: + * + * 1. confirm the source really is deleted. Content and pages are checked against the source + * Management API, because a full pull also leaves out unpublished items; the other kinds are + * pulled as complete lists every run, so absence (or isDeleted) is authoritative — except an + * entirely empty list, which is treated as a bad pull rather than "everything deleted"; + * 2. read the target's current state (content and pages from the API, since this run's pushes + * happened after the pull) and whether it was edited since the last sync — a conflict unless + * --overwrite; + * 3. block the delete when anything that stays on the target still depends on it (a page's + * children, content placed on a page or linked from other content, a container's items, ...); + * 4. refuse the whole delete phase when one kind would lose more than half its mapped entities + * (a wiped or stale cache looks exactly like that), unless --allowMassDeletes. + * + * Deletes run dependents-first (DELETE_ORDER). Each delete — and each mapping whose target is + * already gone — removes its mapping record. Under --preflight the plan is reported and nothing + * is written. + */ +import ansiColors from "ansi-colors"; +import { parse } from "date-fns"; +import { state, getApiClient, getLoggerForGuid } from "../../core/state"; +import { fileOperations } from "../../core/fileOperations"; +import { GuidDataLoader, GuidEntities } from "../pushers/guid-data-loader"; +import { preflightReport } from "../preflight/preflight-report"; +import { deleteUrlRedirections, MAX_URL_REDIRECTION_BATCH_SIZE } from "../pushers/url-redirection-api"; +import { FailureDetail, FailureDetailType, PusherResult } from "../../types/sourceData"; +import { + CandidateFacts, + DeleteCandidate, + DeleteKind, + DELETE_KIND_ELEMENT, + DELETE_KIND_ID_FIELDS, + DELETE_KIND_MAPPING_TYPE, + DELETE_ORDER, + collectFieldReferences, + decideAction, + describeList, + findCandidates, + findMassDeletes, + indexSitemap, + mappingPairs, + MappingPair, + MASS_DELETE_MAX_RATIO, + MASS_DELETE_MIN_COUNT, + modelFieldModelReferences, + pageZoneContentIDs, + pageZoneModuleNames, + toId, +} from "./delete-plan"; + +const DELETED_STATE = 3; +const UNPUBLISHED_STATE = 7; +const API_CONCURRENCY = 6; +const SYSTEM_CONTAINER_NAMES = new Set([ + "agilitycssfiles", + "agilityjavascriptfiles", + "agilityglobalcodetemplates", + "agilitymodulecodetemplates", + "agilitypagecodetemplates", +]); + +const PHASE_LABEL: Record = { + urlRedirection: "Delete URL Redirections", + page: "Delete Pages", + content: "Delete Content", + container: "Delete Containers", + template: "Delete Templates", + model: "Delete Models", + asset: "Delete Assets", + gallery: "Delete Galleries", +}; + +const FAILURE_TYPE: Record = { + urlRedirection: "urlRedirection", + page: "page", + content: "content", + container: "container", + template: "template", + model: "model", + asset: "asset", + gallery: "gallery", +}; + +export interface SyncDeletesOptions { + sourceGuid: string; + targetGuid: string; + locales: string[]; + elements: string[]; +} + +export interface SyncDeletesResult extends PusherResult { + candidates: DeleteCandidate[]; + massDeleteRefused: boolean; +} + +interface LocaleSide { + content: any[]; + pages: any[]; +} + +interface Side { + guid: string; + entities: GuidEntities; + byLocale: Map; + urlRedirectionsPulled: boolean; +} + +/** Everything one planning pass reads, shared by the per-kind planners. */ +interface PlanContext { + opts: SyncDeletesOptions; + source: Side; + target: Side; + overwrite: boolean; + /** Mapping records per kind (and per locale for content/pages). */ + mappings: Map; + /** Target IDs mapped from a live source, per kind/locale — push keeps these in line with the source. */ + liveTargets: Map>; + /** Target IDs this run will delete, per kind/locale (filled in as kinds are planned). */ + deleting: Map>; + /** Live (post-push) target sitemap per locale; null when it could not be read. */ + targetSitemaps: Map; depthOf: Map } | null>; +} + +function groupKey(kind: DeleteKind, locale?: string): string { + return locale ? `${kind}:${locale}` : kind; +} + +function lower(value: any): string { + return typeof value === "string" ? value.toLowerCase() : ""; +} + +function errorStatus(error: any): number | undefined { + return error?.innerError?.response?.status ?? error?.response?.status ?? error?.status; +} + +function errorMessage(error: any): string { + const data = error?.innerError?.response?.data; + if (typeof data === "string" && data) return data; + if (data?.message) return data.message; + return error?.message || String(error); +} + +async function mapPool(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) { + const index = next++; + results[index] = await fn(items[index]); + } + }); + await Promise.all(workers); + return results; +} + +// ─── loading ───────────────────────────────────────────────────────────────── + +async function loadSide(guid: string, locales: string[]): Promise { + const loader = new GuidDataLoader(guid); + const entities = await loader.loadCompleteGuidEntities(locales[0]); + const byLocale = new Map(); + for (const locale of locales) { + const localeEntities = locale === locales[0] ? entities : await loader.loadCompleteGuidEntities(locale); + byLocale.set(locale, { content: localeEntities.content || [], pages: localeEntities.pages || [] }); + } + const redirectionsFile = new fileOperations(guid, locales[0]).readJsonFile("urlredirections/urlredirections.json"); + return { guid, entities, byLocale, urlRedirectionsPulled: !!redirectionsFile }; +} + +function readMappings(kind: DeleteKind, opts: SyncDeletesOptions, locale?: string): any[] { + const fileOps = new fileOperations(opts.targetGuid); + const records = fileOps.getMappingFile(DELETE_KIND_MAPPING_TYPE[kind], opts.sourceGuid, opts.targetGuid, locale ?? null); + return Array.isArray(records) ? records : []; +} + +// ─── API reads ─────────────────────────────────────────────────────────────── + +/** Is a content item / page really deleted in the source, or only missing from the pull? */ +async function confirmSourceDeleted(kind: "content" | "page", sourceID: number, locale: string): Promise { + const api = getApiClient(); + try { + const entity = + kind === "content" + ? await api.contentMethods.getContentItem(sourceID, state.sourceGuid, locale) + : await api.pageMethods.getPage(sourceID, state.sourceGuid, locale); + const itemState = entity?.properties?.state; + if (!entity || itemState === DELETED_STATE) return true; + if (itemState === UNPUBLISHED_STATE) return "unpublished in the source, not deleted"; + return `still exists in the source (state ${itemState}) but is missing from the pull; re-run with --fullPull`; + } catch (error: any) { + if (errorStatus(error) === 404) return true; + return `could not confirm the source delete: ${errorMessage(error)}`; + } +} + +/** The target content item / page as it is now (after this run's pushes). */ +async function readTargetNow( + kind: "content" | "page", + targetID: number, + locale: string +): Promise<{ exists: boolean; entity?: any; error?: string }> { + const api = getApiClient(); + try { + const entity = + kind === "content" + ? await api.contentMethods.getContentItem(targetID, state.targetGuid, locale) + : await api.pageMethods.getPage(targetID, state.targetGuid, locale); + if (!entity || entity?.properties?.state === DELETED_STATE) return { exists: false }; + return { exists: true, entity }; + } catch (error: any) { + if (errorStatus(error) === 404) return { exists: false }; + return { exists: true, error: errorMessage(error) }; + } +} + +// ─── planning ──────────────────────────────────────────────────────────────── + +function newCandidate(kind: DeleteKind, pair: MappingPair, name: string, locale?: string): DeleteCandidate { + return { kind, locale, sourceID: pair.sourceID, targetID: pair.targetID, name, record: pair.record, action: "keep", reason: "" }; +} + +function apply(candidate: DeleteCandidate, facts: CandidateFacts, overwrite: boolean): DeleteCandidate { + const { action, reason } = decideAction(facts, overwrite); + candidate.action = action; + candidate.reason = reason; + return candidate; +} + +/** Is this target entity going away in this run, or kept in line with a live source by the push? */ +function isDeleting(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return ctx.deleting.get(groupKey(kind, locale))?.has(targetID) === true; +} + +function isLiveMapped(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return ctx.liveTargets.get(groupKey(kind, locale))?.has(targetID) === true; +} + +/** A target entity that stays after this run and is not maintained from a live source. */ +function stays(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): boolean { + return !isDeleting(ctx, kind, targetID, locale) && !isLiveMapped(ctx, kind, targetID, locale); +} + +function recordDeleting(ctx: PlanContext, kind: DeleteKind, candidates: DeleteCandidate[], locale?: string) { + const key = groupKey(kind, locale); + const set = ctx.deleting.get(key) ?? new Set(); + candidates.filter((c) => c.action === "delete").forEach((c) => set.add(c.targetID)); + ctx.deleting.set(key, set); +} + +/** + * Re-check blockers until nothing changes: blocking one delete can keep alive something another + * candidate depends on (a page whose child is in conflict, content linked from a kept item). + */ +function settleBlocking( + ctx: PlanContext, + kind: DeleteKind, + candidates: DeleteCandidate[], + blockerOf: (candidate: DeleteCandidate) => string | undefined, + locale?: string +): void { + let changed = true; + while (changed) { + changed = false; + ctx.deleting.set(groupKey(kind, locale), new Set(candidates.filter((c) => c.action === "delete").map((c) => c.targetID))); + for (const candidate of candidates) { + if (candidate.action !== "delete" && candidate.action !== "conflict") continue; + const blocker = blockerOf(candidate); + if (blocker) { + candidate.action = "blocked"; + candidate.reason = blocker; + changed = true; + } + } + } + recordDeleting(ctx, kind, candidates, locale); +} + +function prepareKind( + ctx: PlanContext, + kind: DeleteKind, + liveSourceIDs: Set, + locale?: string +): { pairs: MappingPair[]; candidates: MappingPair[] } { + const records = readMappings(kind, ctx.opts, locale); + ctx.mappings.set(groupKey(kind, locale), records); + const pairs = mappingPairs(kind, records); + const { candidates, liveTargets } = findCandidates(pairs, liveSourceIDs); + ctx.liveTargets.set(groupKey(kind, locale), liveTargets); + return { pairs, candidates }; +} + +function sharedWith(ctx: PlanContext, kind: DeleteKind, targetID: number, locale?: string): number | undefined { + return ctx.liveTargets.get(groupKey(kind, locale))?.get(targetID); +} + +async function planUrlRedirections(ctx: PlanContext): Promise { + const live = new Set((ctx.source.entities.urlRedirections || []).map((r: any) => r?.id).filter(Boolean)); + const { candidates } = prepareKind(ctx, "urlRedirection", live); + const targetIds = new Set((ctx.target.entities.urlRedirections || []).map((r: any) => r?.id)); + const emptySource = !ctx.source.urlRedirectionsPulled || live.size === 0; + return candidates.map((pair) => { + const candidate = newCandidate("urlRedirection", pair, pair.record?.originUrl || `redirection ${pair.targetID}`); + return apply( + candidate, + { + sourceDeleted: emptySource ? "the source URL redirection list is empty or was not pulled; not treating that as deletes" : true, + // An unpulled target list can't prove absence; the delete endpoint reports notFound instead. + targetExists: !ctx.target.urlRedirectionsPulled || targetIds.has(pair.targetID), + targetChanged: false, + sharedWithSourceID: sharedWith(ctx, "urlRedirection", pair.targetID), + }, + ctx.overwrite + ); + }); +} + +async function planPages(ctx: PlanContext, locale: string): Promise { + const sourcePages = ctx.source.byLocale.get(locale)?.pages || []; + const live = new Set(sourcePages.map((p: any) => toId(p?.pageID)).filter(Boolean) as number[]); + const { candidates } = prepareKind(ctx, "page", live, locale); + if (candidates.length === 0) return []; + const targetPagesById = new Map(); + (ctx.target.byLocale.get(locale)?.pages || []).forEach((p: any) => targetPagesById.set(p?.pageID, p)); + + const planned = await mapPool(candidates, API_CONCURRENCY, async (pair) => { + const cached = targetPagesById.get(pair.targetID); + const shared = sharedWith(ctx, "page", pair.targetID, locale); + const sourceDeleted = + live.size === 0 + ? "the source page list for this locale is empty; not treating that as deletes" + : shared !== undefined + ? true + : await confirmSourceDeleted("page", pair.sourceID, locale); + const target = sourceDeleted === true && shared === undefined ? await readTargetNow("page", pair.targetID, locale) : { exists: true }; + const entity = (target as any).entity ?? cached; + const name = entity?.name || entity?.title || `page ${pair.targetID}`; + const candidate = newCandidate("page", pair, name, locale); + const targetVersion = entity?.properties?.versionID ?? 0; + return apply( + candidate, + { + sourceDeleted, + targetExists: target.exists, + targetChanged: targetVersion > (pair.record?.targetVersionID ?? 0), + sharedWithSourceID: shared, + blockedBy: (target as any).error ? `could not read the target page: ${(target as any).error}` : undefined, + }, + ctx.overwrite + ); + }); + + const sitemap = ctx.targetSitemaps.get(locale); + const childrenOf = new Map(); + sitemap?.parentOf.forEach((parent, id) => { + if (parent === null) return; + const list = childrenOf.get(parent) ?? []; + list.push(id); + childrenOf.set(parent, list); + }); + + settleBlocking( + ctx, + "page", + planned, + (candidate) => { + if (!sitemap) return "could not read the target sitemap to check for child pages"; + // The live sitemap reflects this run's page moves, so every child that is not itself being + // deleted keeps the parent — mapped or not. + const remaining = (childrenOf.get(candidate.targetID) || []).filter((id) => !isDeleting(ctx, "page", id, locale)); + if (remaining.length === 0) return undefined; + const names = remaining.map((id) => targetPagesById.get(id)?.name || `page ${id}`); + return `still has child pages on the target: ${describeList(names)}`; + }, + locale + ); + planned.forEach((c) => (c.depth = sitemap?.depthOf.get(c.targetID) ?? 0)); + return planned; +} + +async function planContent(ctx: PlanContext, locale: string): Promise { + const sourceContent = ctx.source.byLocale.get(locale)?.content || []; + const live = new Set(sourceContent.map((c: any) => toId(c?.contentID)).filter(Boolean) as number[]); + const { candidates } = prepareKind(ctx, "content", live, locale); + if (candidates.length === 0) return []; + + const targetContent = ctx.target.byLocale.get(locale)?.content || []; + const targetPages = ctx.target.byLocale.get(locale)?.pages || []; + const contentById = new Map(); + targetContent.forEach((c: any) => contentById.set(c?.contentID, c)); + + // Who points at each content item on the target: other items' fields, and page zones. + const referencedBy = new Map>(); + const addRef = (contentID: number, ref: { kind: "content" | "page"; id: number; name: string }) => { + const list = referencedBy.get(contentID) ?? []; + list.push(ref); + referencedBy.set(contentID, list); + }; + targetContent.forEach((item: any) => { + const name = `${item?.properties?.referenceName || "item"} #${item?.contentID}`; + collectFieldReferences(item?.fields).contentIDs.forEach((id) => { + if (id !== item?.contentID) addRef(id, { kind: "content", id: item.contentID, name }); + }); + }); + targetPages.forEach((page: any) => { + pageZoneContentIDs(page).forEach((id) => addRef(id, { kind: "page", id: page.pageID, name: page?.name || `page ${page.pageID}` })); + }); + + const planned = await mapPool(candidates, API_CONCURRENCY, async (pair) => { + const cached = contentById.get(pair.targetID); + const shared = sharedWith(ctx, "content", pair.targetID, locale); + const sourceDeleted = + live.size === 0 + ? "the source content list for this locale is empty; not treating that as deletes" + : shared !== undefined + ? true + : await confirmSourceDeleted("content", pair.sourceID, locale); + const target = + sourceDeleted === true && shared === undefined ? await readTargetNow("content", pair.targetID, locale) : { exists: true }; + const entity = (target as any).entity ?? cached; + const name = `${entity?.properties?.referenceName || "content"} #${pair.targetID}`; + const candidate = newCandidate("content", pair, name, locale); + const targetVersion = entity?.properties?.versionID ?? 0; + return apply( + candidate, + { + sourceDeleted, + targetExists: target.exists, + targetChanged: targetVersion > (pair.record?.targetVersionID ?? 0), + sharedWithSourceID: shared, + blockedBy: (target as any).error ? `could not read the target item: ${(target as any).error}` : undefined, + }, + ctx.overwrite + ); + }); + + settleBlocking( + ctx, + "content", + planned, + (candidate) => { + const holders = (referencedBy.get(candidate.targetID) || []).filter((ref) => stays(ctx, ref.kind, ref.id, locale)); + if (holders.length === 0) return undefined; + return `still used on the target by ${describeList(holders.map((h) => `${h.kind} ${h.name}`))}`; + }, + locale + ); + return planned; +} + +/** Container (and other guid-level) liveness: in the pulled list and not flagged deleted. */ +function liveIds(list: any[], idOf: (e: any) => any): Set { + const out = new Set(); + (list || []).forEach((e) => { + if (e?.isDeleted === true) return; + const id = toId(idOf(e)); + if (id) out.add(id); + }); + return out; +} + +function emptyListReason(label: string): string { + return `the source ${label} list is empty; not treating that as deletes`; +} + +function changedSince(current: any, recorded: any, format?: string): boolean { + if (!current || !recorded) return false; + const a = format ? parse(current, format, new Date()) : new Date(current); + const b = format ? parse(recorded, format, new Date()) : new Date(recorded); + if (isNaN(a.getTime()) || isNaN(b.getTime())) return false; + return a > b; +} + +function planContainers(ctx: PlanContext): DeleteCandidate[] { + const sourceList = ctx.source.entities.containers || []; + const live = liveIds(sourceList, (c) => c?.contentViewID); + const { candidates } = prepareKind(ctx, "container", live); + const targetById = new Map(); + (ctx.target.entities.containers || []).forEach((c: any) => { + if (c?.isDeleted !== true) targetById.set(c?.contentViewID, c); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const refName = lower(target?.referenceName || pair.record?.targetReferenceName); + const candidate = newCandidate("container", pair, target?.referenceName || pair.record?.targetReferenceName || `container ${pair.targetID}`); + + let blockedBy: string | undefined; + if (SYSTEM_CONTAINER_NAMES.has(refName)) blockedBy = "system container"; + if (!blockedBy && target) { + const holders: string[] = []; + ctx.opts.locales.forEach((locale) => { + const side = ctx.target.byLocale.get(locale); + (side?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + const itemName = `content #${item.contentID} [${locale}]`; + if (lower(item?.properties?.referenceName) === refName) holders.push(itemName); + else if (collectFieldReferences(item?.fields).containers.has(refName)) holders.push(`${itemName} (linked list)`); + }); + (side?.pages || []).forEach((page: any) => { + if (lower(page?.dynamic?.referenceName) === refName && stays(ctx, "page", page.pageID, locale)) { + holders.push(`dynamic page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + if (holders.length > 0) blockedBy = `still used on the target by ${describeList(holders)}`; + } + + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("container") : true, + targetExists: !!target, + targetChanged: changedSince(target?.lastModifiedDate, pair.record?.targetLastModifiedDate, "MM/dd/yyyy hh:mma"), + sharedWithSourceID: sharedWith(ctx, "container", pair.targetID), + blockedBy, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "container", planned); + return planned; +} + +function planTemplates(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.templates || [], (t) => t?.pageTemplateID); + const { candidates } = prepareKind(ctx, "template", live); + const targetById = new Map(); + (ctx.target.entities.templates || []).forEach((t: any) => { + if (t?.isDeleted !== true) targetById.set(t?.pageTemplateID, t); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const templateName = target?.pageTemplateName || pair.record?.targetPageTemplateName; + const candidate = newCandidate("template", pair, templateName || `template ${pair.targetID}`); + const holders: string[] = []; + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.pages || []).forEach((page: any) => { + if (lower(page?.templateName) === lower(templateName) && stays(ctx, "page", page.pageID, locale)) { + holders.push(`page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("page template") : true, + targetExists: !!target, + targetChanged: false, + sharedWithSourceID: sharedWith(ctx, "template", pair.targetID), + blockedBy: holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "template", planned); + return planned; +} + +function planModels(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.models || [], (m) => m?.id); + const { candidates } = prepareKind(ctx, "model", live); + const targetModels = ctx.target.entities.models || []; + const targetById = new Map(); + targetModels.forEach((m: any) => targetById.set(m?.id, m)); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const refName = target?.referenceName || pair.record?.targetReferenceName; + const candidate = newCandidate("model", pair, refName || `model ${pair.targetID}`); + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("model") : true, + targetExists: !!target, + targetChanged: changedSince(target?.lastModifiedDate, pair.record?.targetLastModifiedDate), + sharedWithSourceID: sharedWith(ctx, "model", pair.targetID), + }, + ctx.overwrite + ); + }); + + settleBlocking(ctx, "model", planned, (candidate) => { + const target = targetById.get(candidate.targetID); + const refName = lower(target?.referenceName || candidate.record?.targetReferenceName); + const holders: string[] = []; + (ctx.target.entities.containers || []).forEach((c: any) => { + if (c?.isDeleted === true || c?.contentDefinitionID !== candidate.targetID) return; + if (stays(ctx, "container", c.contentViewID)) holders.push(`container ${c.referenceName}`); + }); + ctx.opts.locales.forEach((locale) => { + const side = ctx.target.byLocale.get(locale); + (side?.content || []).forEach((item: any) => { + if (lower(item?.properties?.definitionName) === refName && stays(ctx, "content", item.contentID, locale)) { + holders.push(`content #${item.contentID} [${locale}]`); + } + }); + (side?.pages || []).forEach((page: any) => { + if (pageZoneModuleNames(page).includes(refName) && stays(ctx, "page", page.pageID, locale)) { + holders.push(`page ${page?.name || page.pageID} [${locale}]`); + } + }); + }); + targetModels.forEach((m: any) => { + if (m?.id === candidate.targetID || !stays(ctx, "model", m?.id)) return; + if (modelFieldModelReferences(m).includes(refName)) holders.push(`model ${m.referenceName}`); + }); + return holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined; + }); + return planned; +} + +function planAssets(ctx: PlanContext): DeleteCandidate[] { + const isFile = (a: any) => a?.isFolder !== true; + const live = liveIds((ctx.source.entities.assets || []).filter(isFile), (a) => a?.mediaID); + const { candidates } = prepareKind(ctx, "asset", live); + const targetById = new Map(); + (ctx.target.entities.assets || []).forEach((a: any) => { + if (a?.isDeleted !== true && isFile(a)) targetById.set(a?.mediaID, a); + }); + + // Content that stays, as one searchable string per item, built once. + const remainingContent: Array<{ name: string; text: string }> = []; + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + remainingContent.push({ name: `content #${item.contentID} [${locale}]`, text: JSON.stringify(item?.fields ?? {}) }); + }); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const candidate = newCandidate("asset", pair, target?.fileName || pair.record?.targetUrl || `asset ${pair.targetID}`); + let blockedBy: string | undefined; + if (target) { + const urls = [target.url, target.originUrl, target.edgeUrl, pair.record?.targetUrl].filter( + (u): u is string => typeof u === "string" && u.length > 0 + ); + const holders = remainingContent.filter((c) => urls.some((u) => c.text.includes(u))).map((c) => c.name); + if (holders.length > 0) blockedBy = `still used on the target by ${describeList(holders)}`; + } + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("asset") : true, + targetExists: !!target, + targetChanged: changedSince(target?.dateModified, pair.record?.targetDateModified), + sharedWithSourceID: sharedWith(ctx, "asset", pair.targetID), + blockedBy, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "asset", planned); + return planned; +} + +function planGalleries(ctx: PlanContext): DeleteCandidate[] { + const live = liveIds(ctx.source.entities.galleries || [], (g) => g?.mediaGroupingID); + const { candidates } = prepareKind(ctx, "gallery", live); + const targetById = new Map(); + (ctx.target.entities.galleries || []).forEach((g: any) => { + if (g?.isDeleted !== true) targetById.set(g?.mediaGroupingID, g); + }); + + const planned = candidates.map((pair) => { + const target = targetById.get(pair.targetID); + const candidate = newCandidate("gallery", pair, target?.name || `gallery ${pair.targetID}`); + const holders: string[] = []; + if (target) { + (ctx.target.entities.assets || []).forEach((a: any) => { + if (a?.isDeleted === true || a?.mediaGroupingID !== pair.targetID) return; + if (stays(ctx, "asset", a.mediaID)) holders.push(`asset ${a.fileName || a.mediaID}`); + }); + ctx.opts.locales.forEach((locale) => { + (ctx.target.byLocale.get(locale)?.content || []).forEach((item: any) => { + if (!stays(ctx, "content", item.contentID, locale)) return; + if (collectFieldReferences(item?.fields).galleryIDs.has(pair.targetID)) { + holders.push(`content #${item.contentID} [${locale}]`); + } + }); + }); + } + return apply( + candidate, + { + sourceDeleted: live.size === 0 ? emptyListReason("gallery") : true, + targetExists: !!target, + targetChanged: changedSince(target?.modifiedOn, pair.record?.targetModifiedOn, "MM/dd/yyyy hh:mma"), + sharedWithSourceID: sharedWith(ctx, "gallery", pair.targetID), + blockedBy: holders.length > 0 ? `still used on the target by ${describeList(holders)}` : undefined, + }, + ctx.overwrite + ); + }); + recordDeleting(ctx, "gallery", planned); + return planned; +} + +async function loadTargetSitemaps(ctx: PlanContext): Promise { + const api = getApiClient(); + for (const locale of ctx.opts.locales) { + try { + const channels = await api.pageMethods.getSitemap(ctx.opts.targetGuid, locale); + ctx.targetSitemaps.set(locale, indexSitemap(channels as any[])); + } catch { + ctx.targetSitemaps.set(locale, null); + } + } +} + +/** Build the full delete plan. Exported for tests; reads the pulled caches and the mapping files. */ +export async function planSyncDeletes(opts: SyncDeletesOptions): Promise<{ groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }> }> { + const ctx: PlanContext = { + opts, + source: await loadSide(opts.sourceGuid, opts.locales), + target: await loadSide(opts.targetGuid, opts.locales), + overwrite: state.overwrite === true, + mappings: new Map(), + liveTargets: new Map(), + deleting: new Map(), + targetSitemaps: new Map(), + }; + const wants = (kind: DeleteKind) => opts.elements.includes(DELETE_KIND_ELEMENT[kind]); + const groups: Array<{ kind: DeleteKind; locale?: string; mapped: number; candidates: DeleteCandidate[] }> = []; + const add = (kind: DeleteKind, candidates: DeleteCandidate[], locale?: string) => { + groups.push({ kind, locale, mapped: (ctx.mappings.get(groupKey(kind, locale)) || []).length, candidates }); + }; + + if (wants("page")) await loadTargetSitemaps(ctx); + + // Planned in execution order: each kind's blockers look at what the kinds before it delete. + if (wants("urlRedirection")) add("urlRedirection", await planUrlRedirections(ctx)); + for (const locale of opts.locales) { + if (wants("page")) add("page", await planPages(ctx, locale), locale); + } + for (const locale of opts.locales) { + if (wants("content")) add("content", await planContent(ctx, locale), locale); + } + if (wants("container")) add("container", planContainers(ctx)); + if (wants("template")) add("template", planTemplates(ctx)); + if (wants("model")) add("model", planModels(ctx)); + if (wants("asset")) add("asset", planAssets(ctx)); + if (wants("gallery")) add("gallery", planGalleries(ctx)); + + return { groups }; +} + +// ─── execution ─────────────────────────────────────────────────────────────── + +async function waitForBatch(batchID: number, guid: string): Promise { + const api = getApiClient(); + for (let attempt = 0; attempt < 150; attempt++) { + const batch: any = await api.batchMethods.getBatch(batchID, guid, true); + const batchState = batch?.batchState ?? batch?.BatchState; + if (batchState === 3) { + const errorData = batch?.errorData ?? batch?.ErrorData; + const itemError = (batch?.items ?? batch?.Items ?? []).find((i: any) => i?.errorMessage)?.errorMessage; + if (errorData || itemError) throw new Error(`batch ${batchID} failed: ${itemError || errorData}`); + return; + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + throw new Error(`batch ${batchID} did not finish in time; check the Batches page in the Content Manager`); +} + +async function deleteOne(candidate: DeleteCandidate, opts: SyncDeletesOptions): Promise { + const api = getApiClient(); + const guid = state.targetGuid; + switch (candidate.kind) { + case "page": + case "content": { + // Deleting a page can take its zone content with it, and a delete in one locale can reach + // another: re-read right before deleting so an already-gone target counts as done. + const now = await readTargetNow(candidate.kind, candidate.targetID, candidate.locale!); + if (!now.exists) return; + break; + } + } + switch (candidate.kind) { + case "page": { + const [batchID] = await api.pageMethods.deletePage(candidate.targetID, guid, candidate.locale!, "Deleted by agility-cli --syncDeletes", true); + await waitForBatch(batchID, guid); + return; + } + case "content": { + const [batchID] = await api.contentMethods.deleteContent(candidate.targetID, guid, candidate.locale!, "Deleted by agility-cli --syncDeletes", true); + await waitForBatch(batchID, guid); + return; + } + case "container": + await api.containerMethods.deleteContainer(candidate.targetID, guid); + return; + case "template": + await api.pageMethods.deletePageTemplate(guid, opts.locales[0], candidate.targetID); + return; + case "model": + await api.modelMethods.deleteModel(candidate.targetID, guid); + return; + case "asset": + await api.assetMethods.deleteFile(candidate.targetID, guid); + return; + case "gallery": + await api.assetMethods.deleteGallery(guid, candidate.targetID); + return; + default: + throw new Error(`unsupported delete kind ${candidate.kind}`); + } +} + +/** Execute one group's deletes; returns the candidates whose delete failed. */ +async function executeGroup(candidates: DeleteCandidate[], opts: SyncDeletesOptions): Promise> { + const failures = new Map(); + const toDelete = candidates.filter((c) => c.action === "delete"); + if (toDelete.length === 0) return failures; + const kind = toDelete[0].kind; + + if (kind === "urlRedirection") { + for (let i = 0; i < toDelete.length; i += MAX_URL_REDIRECTION_BATCH_SIZE) { + const chunk = toDelete.slice(i, i + MAX_URL_REDIRECTION_BATCH_SIZE); + try { + // notFound means it is already gone, which is the outcome we wanted. + await deleteUrlRedirections(state.targetGuid, chunk.map((c) => c.targetID)); + } catch (error: any) { + chunk.forEach((c) => failures.set(c, errorMessage(error))); + } + } + return failures; + } + + if (kind === "page") { + // Children before parents: a parent is only deletable once its children are gone. + const ordered = [...toDelete].sort((a, b) => (b.depth ?? 0) - (a.depth ?? 0)); + for (const candidate of ordered) { + try { + await deleteOne(candidate, opts); + } catch (error: any) { + failures.set(candidate, errorMessage(error)); + } + } + return failures; + } + + const limit = kind === "content" ? API_CONCURRENCY : 1; + await mapPool(toDelete, limit, async (candidate) => { + try { + await deleteOne(candidate, opts); + } catch (error: any) { + failures.set(candidate, errorMessage(error)); + } + }); + return failures; +} + +/** Drop the mapping records of deleted / unmapped candidates from one kind's mapping file. */ +function removeMappings(kind: DeleteKind, removed: DeleteCandidate[], opts: SyncDeletesOptions, locale?: string): void { + if (removed.length === 0 || state.preflight) return; + const fields = DELETE_KIND_ID_FIELDS[kind]; + const drop = new Set(removed.map((c) => `${c.sourceID}:${c.targetID}`)); + const records = readMappings(kind, opts, locale); + const kept = records.filter((r) => !drop.has(`${toId(r?.[fields.source])}:${toId(r?.[fields.target])}`)); + if (kept.length === records.length) return; + new fileOperations(opts.targetGuid).saveMappingFile(kept, DELETE_KIND_MAPPING_TYPE[kind], opts.sourceGuid, opts.targetGuid, locale ?? null); +} + +function preflightPhase(kind: DeleteKind): string { + return PHASE_LABEL[kind]; +} + +function recordPreflight(candidate: DeleteCandidate): void { + const base = { phase: preflightPhase(candidate.kind), name: candidate.name, locale: candidate.locale, detail: candidate.reason }; + switch (candidate.action) { + case "delete": + preflightReport.record({ ...base, action: "delete" }); + break; + case "conflict": + preflightReport.record({ ...base, action: "conflict" }); + break; + default: + preflightReport.record({ ...base, action: "skip" }); + } +} + +/** + * Run the delete phase. Called by the push orchestrator after every pusher, only with --syncDeletes. + */ +export async function runSyncDeletes(opts: SyncDeletesOptions): Promise { + const logger = getLoggerForGuid(opts.sourceGuid); + const result: SyncDeletesResult = { + successful: 0, + failed: 0, + skipped: 0, + status: "success", + failureDetails: [], + warningDetails: [], + candidates: [], + massDeleteRefused: false, + }; + + console.log(ansiColors.cyan("\n🗑 Sync deletes: checking for entities deleted in the source since the last sync...")); + + const { groups } = await planSyncDeletes(opts); + result.candidates = groups.reduce((all, g) => all.concat(g.candidates), []); + + const violations = findMassDeletes(groups); + if (violations.length > 0 && !state.allowMassDeletes) { + result.massDeleteRefused = true; + const lines = violations.map( + (v) => `${PHASE_LABEL[v.kind]}${v.locale ? ` [${v.locale}]` : ""}: ${v.deletes} of ${v.mapped} mapped entities` + ); + const message = + `Refusing to sync deletes: more than ${Math.round(MASS_DELETE_MAX_RATIO * 100)}% (and more than ${MASS_DELETE_MIN_COUNT}) ` + + `of the mapped entities would be deleted — ${lines.join("; ")}. This usually means a stale or partial local cache ` + + `(re-run with --fullPull). If the deletes are intended, re-run with --allowMassDeletes.`; + console.log(ansiColors.red(`\n❌ ${message}`)); + logger?.error(message); + result.failed = 1; + result.status = "error"; + result.failureDetails!.push({ name: "sync deletes", error: message }); + if (state.preflight) result.candidates.forEach(recordPreflight); + return result; + } + + for (const kind of DELETE_ORDER) { + for (const group of groups.filter((g) => g.kind === kind)) { + const { candidates, locale } = group; + if (candidates.length === 0) continue; + + if (state.preflight) { + candidates.forEach(recordPreflight); + candidates.forEach((c) => (c.action === "delete" ? result.successful++ : result.skipped++)); + continue; + } + + const failures = await executeGroup(candidates, opts); + const removed: DeleteCandidate[] = []; + + for (const candidate of candidates) { + const where = candidate.locale ? ` [${candidate.locale}]` : ""; + const label = `${PHASE_LABEL[kind].replace("Delete ", "").replace(/s$/, "")} ${candidate.name}${where}`; + const failure = failures.get(candidate); + if (candidate.action === "delete" && !failure) { + result.successful++; + removed.push(candidate); + logger?.success(`🗑 Deleted ${label} (target ID ${candidate.targetID}) — ${candidate.reason}`); + console.log(ansiColors.green(` ✓ deleted ${label}`)); + } else if (candidate.action === "delete") { + result.failed++; + result.status = "error"; + result.failureDetails!.push({ name: candidate.name, error: `delete failed: ${failure}`, type: FAILURE_TYPE[kind], locale: candidate.locale, guid: opts.targetGuid }); + logger?.error(`Failed to delete ${label} (target ID ${candidate.targetID}): ${failure}`); + console.log(ansiColors.red(` ✗ ${label}: delete failed — ${failure}`)); + } else if (candidate.action === "unmap") { + result.skipped++; + removed.push(candidate); + logger?.info(`${label}: ${candidate.reason}`); + } else if (candidate.action === "conflict" || candidate.action === "blocked") { + result.skipped++; + const detail: FailureDetail = { name: candidate.name, error: `not deleted — ${candidate.reason}`, type: FAILURE_TYPE[kind], locale: candidate.locale }; + result.warningDetails!.push(detail); + logger?.warning(`${label}: not deleted — ${candidate.reason}`); + console.log(ansiColors.yellow(` ⚠ ${label}: not deleted — ${candidate.reason}`)); + } else { + result.skipped++; + logger?.info(`${label}: ${candidate.reason}`); + } + } + + removeMappings(kind, removed, opts, locale); + } + } + + const planned = result.candidates.length; + console.log( + ansiColors.gray(`\nSync deletes: `) + + ansiColors.green(`${result.successful} deleted, `) + + ansiColors.yellow(`${result.skipped} skipped, `) + + (result.failed > 0 ? ansiColors.red : ansiColors.gray)(`${result.failed} failed`) + + ansiColors.gray(` (${planned} mapping(s) whose source is gone)\n`) + ); + + return result; +} diff --git a/src/lib/deleters/tests/delete-plan.test.ts b/src/lib/deleters/tests/delete-plan.test.ts new file mode 100644 index 0000000..d26916d --- /dev/null +++ b/src/lib/deleters/tests/delete-plan.test.ts @@ -0,0 +1,177 @@ +import { + collectFieldReferences, + decideAction, + DeleteCandidate, + findCandidates, + findMassDeletes, + indexSitemap, + mappingPairs, + modelFieldModelReferences, + pageZoneContentIDs, + pageZoneModuleNames, +} from "../delete-plan"; + +const facts = (over: Partial[0]> = {}) => ({ + sourceDeleted: true as const, + targetExists: true, + targetChanged: false, + ...over, +}); + +describe("decideAction", () => { + it("deletes when the source is deleted and the target is untouched", () => { + expect(decideAction(facts(), false).action).toBe("delete"); + }); + + it("keeps everything when the source delete is not confirmed", () => { + const result = decideAction(facts({ sourceDeleted: "unpublished in the source, not deleted" }), true); + expect(result).toEqual({ action: "keep", reason: "unpublished in the source, not deleted" }); + }); + + it("only drops the mapping when a live source still owns the target", () => { + const result = decideAction(facts({ sharedWithSourceID: 42, targetChanged: true }), true); + expect(result.action).toBe("unmap"); + expect(result.reason).toContain("42"); + }); + + it("only drops the mapping when the target is already gone", () => { + expect(decideAction(facts({ targetExists: false }), false).action).toBe("unmap"); + }); + + it("is a conflict when the target changed and --overwrite is off", () => { + expect(decideAction(facts({ targetChanged: true }), false).action).toBe("conflict"); + }); + + it("deletes a changed target under --overwrite", () => { + const result = decideAction(facts({ targetChanged: true }), true); + expect(result.action).toBe("delete"); + expect(result.reason).toContain("--overwrite"); + }); + + it("blocks before anything else that would delete, even under --overwrite", () => { + const result = decideAction(facts({ targetChanged: true, blockedBy: "still used by page home" }), true); + expect(result).toEqual({ action: "blocked", reason: "still used by page home" }); + }); +}); + +describe("mappingPairs / findCandidates", () => { + const records = [ + { sourceContentID: 1, targetContentID: 101 }, + { sourceContentID: 2, targetContentID: 102 }, + { sourceContentID: 3, targetContentID: 101 }, // dead duplicate onto a live target + { sourceContentID: null, targetContentID: 103 }, // unusable + { sourceContentID: "4", targetContentID: "104" }, // string ids still count + ]; + + it("reads each kind's id fields and drops unusable records", () => { + const pairs = mappingPairs("content", records); + expect(pairs.map((p) => [p.sourceID, p.targetID])).toEqual([ + [1, 101], + [2, 102], + [3, 101], + [4, 104], + ]); + }); + + it("returns records whose source is gone, and the targets still owned by a live source", () => { + const { candidates, liveTargets } = findCandidates(mappingPairs("content", records), new Set([1])); + expect(candidates.map((c) => c.sourceID)).toEqual([2, 3, 4]); + expect(liveTargets.get(101)).toBe(1); + expect(liveTargets.has(102)).toBe(false); + }); + + it("uses the guid-level field names", () => { + expect(mappingPairs("model", [{ sourceID: 5, targetID: 50 }])[0]).toMatchObject({ sourceID: 5, targetID: 50 }); + expect(mappingPairs("container", [{ sourceContentViewID: 6, targetContentViewID: 60 }])).toHaveLength(1); + expect(mappingPairs("gallery", [{ sourceMediaGroupingID: 7, targetMediaGroupingID: 70 }])).toHaveLength(1); + expect(mappingPairs("urlRedirection", [{ sourceUrlRedirectionID: 8, targetUrlRedirectionID: 80 }])).toHaveLength(1); + }); +}); + +describe("findMassDeletes", () => { + const candidates = (n: number, action: DeleteCandidate["action"] = "delete"): DeleteCandidate[] => + Array.from({ length: n }, (_, i) => ({ + kind: "content", + sourceID: i + 1, + targetID: i + 1001, + name: `item ${i}`, + record: {}, + action, + reason: "", + })); + + it("flags a kind losing more than half of many mapped entities", () => { + expect(findMassDeletes([{ kind: "content", locale: "en-us", mapped: 20, candidates: candidates(11) }])).toEqual([ + { kind: "content", locale: "en-us", deletes: 11, mapped: 20 }, + ]); + }); + + it("allows small deletes even when they are most of the kind", () => { + expect(findMassDeletes([{ kind: "model", mapped: 4, candidates: candidates(4) }])).toEqual([]); + }); + + it("allows many deletes when they are a minority", () => { + expect(findMassDeletes([{ kind: "content", mapped: 100, candidates: candidates(40) }])).toEqual([]); + }); + + it("counts only actual deletes", () => { + expect(findMassDeletes([{ kind: "content", mapped: 20, candidates: candidates(15, "conflict") }])).toEqual([]); + }); +}); + +describe("collectFieldReferences", () => { + it("finds linked content in every shape", () => { + const refs = collectFieldReferences({ + single: { contentid: 11, fulllist: false }, + list: { referencename: "Posts_List", sortids: "12,13" }, + upper: { ContentID: "14" }, + scalar: "15", + csv: "16, 17", + nested: [{ inner: { contentID: 18 } }], + text: "Hello 19 world", + }); + expect(Array.from(refs.contentIDs).sort((a, b) => a - b)).toEqual([11, 12, 13, 14, 15, 16, 17, 18]); + expect(Array.from(refs.containers)).toEqual(["posts_list"]); + }); + + it("finds galleries", () => { + expect(Array.from(collectFieldReferences({ photos: { galleryid: 3 } }).galleryIDs)).toEqual([3]); + }); + + it("tolerates missing fields", () => { + expect(collectFieldReferences(undefined).contentIDs.size).toBe(0); + }); +}); + +describe("page and model helpers", () => { + const page = { + zones: { + main: [ + { module: "RichText", item: { contentid: 5 } }, + { module: "Hero", item: { contentID: "6" } }, + ], + side: [{ module: "Nav", item: {} }], + }, + }; + + it("reads zone content and module names", () => { + expect(pageZoneContentIDs(page)).toEqual([5, 6]); + expect(pageZoneModuleNames(page)).toEqual(["richtext", "hero", "nav"]); + }); + + it("reads model-to-model field references", () => { + const model = { fields: [{ settings: { ContentDefinition: "Author" } }, { settings: {} }] }; + expect(modelFieldModelReferences(model)).toEqual(["author"]); + }); + + it("indexes a sitemap into parents and depths", () => { + const { parentOf, depthOf } = indexSitemap([ + { pages: [{ pageID: 1, childPages: [{ pageID: 2, childPages: [{ pageID: 3, childPages: [] }] }] }] }, + { pages: [{ pageID: 9, childPages: null }] }, + ]); + expect(parentOf.get(1)).toBeNull(); + expect(parentOf.get(3)).toBe(2); + expect(depthOf.get(3)).toBe(2); + expect(depthOf.get(9)).toBe(0); + }); +}); diff --git a/src/lib/deleters/tests/sync-deletes.test.ts b/src/lib/deleters/tests/sync-deletes.test.ts new file mode 100644 index 0000000..3a657a6 --- /dev/null +++ b/src/lib/deleters/tests/sync-deletes.test.ts @@ -0,0 +1,358 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +jest.mock("../../pushers/url-redirection-api", () => ({ + MAX_URL_REDIRECTION_BATCH_SIZE: 250, + deleteUrlRedirections: jest.fn(async (_guid: string, ids: number[]) => ({ deleted: ids, notFound: [] })), +})); + +import { state, resetState, setState } from "../../../core/state"; +import { preflightReport } from "../../preflight/preflight-report"; +import { runSyncDeletes } from "../sync-deletes"; +import { deleteUrlRedirections } from "../../pushers/url-redirection-api"; + +const SRC = "src-1"; +const TGT = "tgt-1"; +const LOCALE = "en-us"; + +function writeJson(file: string, data: any) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify(data)); +} + +function notFound(): any { + const error: any = new Error("Unable to retrieve"); + error.innerError = { response: { status: 404, data: "not found" } }; + return error; +} + +interface Fixture { + root: string; + deleted: string[]; + api: any; +} + +/** + * One source/target pair with a mapped entity of every kind, some deleted in the source. + * + * Source IDs are n, target IDs are 100 + n; target-only entities are 900+. + */ +function buildFixture(): Fixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "sync-deletes-")); + const src = (rel: string, data: any) => writeJson(path.join(root, SRC, rel), data); + const tgt = (rel: string, data: any) => writeJson(path.join(root, TGT, rel), data); + const map = (rel: string, data: any[]) => writeJson(path.join(root, "mappings", `${SRC}-${TGT}`, rel, "mappings.json"), data); + + // ── source: only the live entities remain after the pull ── + src(`${LOCALE}/item/1.json`, { contentID: 1, properties: { state: 2, versionID: 1, referenceName: "posts" } }); + src(`${LOCALE}/page/10.json`, { pageID: 10, name: "home", properties: { state: 2, versionID: 1 }, zones: {} }); + src("models/20.json", { id: 20, referenceName: "Post", lastModifiedDate: "2026-01-01T00:00:00" }); + src("containers/30.json", { contentViewID: 30, referenceName: "posts", lastModifiedDate: "01/01/2026 10:00AM" }); + src("templates/40.json", { pageTemplateID: 40, pageTemplateName: "Main" }); + src("assets/json/1.json", { assetMedias: [{ mediaID: 50, fileName: "a.png", dateModified: "2026-01-01T00:00:00" }] }); + src("galleries/60.json", { mediaGroupingID: 60, name: "Live", modifiedOn: "01/01/2026 10:00AM" }); + src(`${LOCALE}/urlredirections/urlredirections.json`, { items: [{ id: 70, originUrl: "/live" }] }); + + // ── target ── + const item = (id: number, extra: any = {}) => ({ + contentID: id, + properties: { state: 2, versionID: 1, referenceName: "posts", definitionName: "Post" }, + fields: {}, + ...extra, + }); + [101, 102, 103, 104, 105].forEach((id) => tgt(`${LOCALE}/item/${id}.json`, item(id))); + // target-only item linking to 105 + tgt(`${LOCALE}/item/900.json`, item(900, { fields: { related: { contentid: 105 } } })); + [110, 111, 112, 113].forEach((id) => + tgt(`${LOCALE}/page/${id}.json`, { pageID: id, name: `page-${id}`, templateName: "Main", properties: { versionID: 1 }, zones: {} }) + ); + tgt(`${LOCALE}/page/950.json`, { pageID: 950, name: "native-child", templateName: "Main", properties: { versionID: 1 }, zones: {} }); + tgt(`${LOCALE}/page/960.json`, { pageID: 960, name: "native-page", templateName: "Old", properties: { versionID: 1 }, zones: {} }); + tgt("models/120.json", { id: 120, referenceName: "Post", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("models/121.json", { id: 121, referenceName: "Gone", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("models/122.json", { id: 122, referenceName: "InUse", lastModifiedDate: "2026-01-01T00:00:00" }); + tgt("containers/130.json", { contentViewID: 130, referenceName: "posts", contentDefinitionID: 120, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("containers/131.json", { contentViewID: 131, referenceName: "empty", contentDefinitionID: 121, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("containers/990.json", { contentViewID: 990, referenceName: "native", contentDefinitionID: 122, lastModifiedDate: "01/01/2026 10:00AM" }); + tgt("templates/140.json", { pageTemplateID: 140, pageTemplateName: "Main" }); + tgt("templates/141.json", { pageTemplateID: 141, pageTemplateName: "Old" }); + tgt("assets/json/1.json", { + assetMedias: [ + { mediaID: 150, fileName: "a.png", dateModified: "2026-01-01T00:00:00" }, + { mediaID: 151, fileName: "b.png", url: "https://cdn/b.png", dateModified: "2026-01-01T00:00:00" }, + ], + }); + tgt("galleries/160.json", { mediaGroupingID: 160, name: "Live", modifiedOn: "01/01/2026 10:00AM" }); + tgt("galleries/161.json", { mediaGroupingID: 161, name: "Gone", modifiedOn: "01/01/2026 10:00AM" }); + tgt(`${LOCALE}/urlredirections/urlredirections.json`, { items: [{ id: 170 }, { id: 171 }] }); + + // ── mappings ── + map(`${LOCALE}/item`, [1, 2, 3, 4, 5].map((n) => ({ sourceContentID: n, targetContentID: 100 + n, sourceVersionID: 1, targetVersionID: 1 }))); + map(`${LOCALE}/page`, [10, 11, 12, 13].map((n) => ({ sourcePageID: n, targetPageID: 100 + n, sourceVersionID: 1, targetVersionID: 1 }))); + map("models", [20, 21, 22].map((n) => ({ sourceID: n, targetID: 100 + n, targetLastModifiedDate: "2026-01-01T00:00:00" }))); + map("containers", [30, 31].map((n) => ({ sourceContentViewID: n, targetContentViewID: 100 + n, targetLastModifiedDate: "01/01/2026 10:00AM" }))); + map("templates", [40, 41].map((n) => ({ sourcePageTemplateID: n, targetPageTemplateID: 100 + n }))); + map("assets", [50, 51].map((n) => ({ sourceMediaID: n, targetMediaID: 100 + n, targetDateModified: "2026-01-01T00:00:00" }))); + map("galleries", [60, 61].map((n) => ({ sourceMediaGroupingID: n, targetMediaGroupingID: 100 + n, targetModifiedOn: "01/01/2026 10:00AM" }))); + map("urlredirections", [70, 71].map((n) => ({ sourceUrlRedirectionID: n, targetUrlRedirectionID: 100 + n, originUrl: `/r${n}` }))); + + const deleted: string[] = []; + const targetVersion: Record = { 104: 9 }; // 104 was edited on the target since the sync + const api = { + contentMethods: { + getContentItem: jest.fn(async (id: number, guid: string) => { + if (guid === SRC) { + if (id === 3) return { contentID: 3, properties: { state: 7 } }; + throw notFound(); + } + if (deleted.includes(`content:${id}`)) throw notFound(); + return { contentID: id, properties: { state: 2, versionID: targetVersion[id] ?? 1, referenceName: "posts" } }; + }), + deleteContent: jest.fn(async (id: number) => { + deleted.push(`content:${id}`); + return [1]; + }), + }, + pageMethods: { + getPage: jest.fn(async (id: number, guid: string) => { + if (guid === SRC) throw notFound(); + if (deleted.includes(`page:${id}`)) throw notFound(); + return { pageID: id, name: `page-${id}`, properties: { state: 2, versionID: 1 } }; + }), + getSitemap: jest.fn(async () => [ + { + pages: [ + { pageID: 110, childPages: [] }, + { pageID: 111, childPages: [{ pageID: 112, childPages: [] }] }, + { pageID: 113, childPages: [{ pageID: 950, childPages: [] }] }, + { pageID: 960, childPages: [] }, + ], + }, + ]), + deletePage: jest.fn(async (id: number) => { + deleted.push(`page:${id}`); + return [2]; + }), + deletePageTemplate: jest.fn(async (_guid: string, _locale: string, id: number) => { + deleted.push(`template:${id}`); + return "ok"; + }), + }, + batchMethods: { getBatch: jest.fn(async () => ({ batchState: 3, items: [] })) }, + containerMethods: { + deleteContainer: jest.fn(async (id: number) => { + deleted.push(`container:${id}`); + return "ok"; + }), + }, + modelMethods: { + deleteModel: jest.fn(async (id: number) => { + deleted.push(`model:${id}`); + return "ok"; + }), + }, + assetMethods: { + deleteFile: jest.fn(async (id: number) => { + deleted.push(`asset:${id}`); + return "ok"; + }), + deleteGallery: jest.fn(async (_guid: string, id: number) => { + deleted.push(`gallery:${id}`); + return "ok"; + }), + }, + }; + + return { root, deleted, api }; +} + +const ALL_ELEMENTS = ["Galleries", "Assets", "Models", "Containers", "Content", "Templates", "Pages", "UrlRedirections"]; + +function configure(fixture: Fixture, extra: Record = {}) { + resetState(); + setState({ rootPath: fixture.root, sourceGuid: SRC, targetGuid: TGT, locales: LOCALE, isSync: true, syncDeletes: true, ...extra }); + state.cachedApiClient = fixture.api; + preflightReport.reset(); +} + +function run() { + return runSyncDeletes({ sourceGuid: SRC, targetGuid: TGT, locales: [LOCALE], elements: ALL_ELEMENTS }); +} + +function readMapping(fixture: Fixture, rel: string): any[] { + return JSON.parse(fs.readFileSync(path.join(fixture.root, "mappings", `${SRC}-${TGT}`, rel, "mappings.json"), "utf8")); +} + +function actions(result: Awaited>) { + const out: Record = {}; + result.candidates.forEach((c) => (out[`${c.kind}:${c.targetID}`] = c.action)); + return out; +} + +let logSpy: jest.SpyInstance; +beforeEach(() => { + logSpy = jest.spyOn(console, "log").mockImplementation(() => undefined); + (deleteUrlRedirections as jest.Mock).mockClear(); +}); +afterEach(() => logSpy.mockRestore()); + +describe("runSyncDeletes", () => { + it("plans every kind: deletes what is safe, keeps what is unpublished, changed or still used", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await run(); + + expect(actions(result)).toEqual({ + "urlRedirection:171": "delete", + "page:111": "delete", + "page:112": "delete", + "page:113": "blocked", // native child page 950 stays + "content:102": "delete", + "content:103": "keep", // unpublished in the source + "content:104": "conflict", // edited on the target + "content:105": "blocked", // linked from native item 900 + "container:131": "delete", + "template:141": "blocked", // native page 960 uses it + "model:121": "delete", + "model:122": "blocked", // native container 990 uses it + "asset:151": "delete", + "gallery:161": "delete", + }); + expect(result.candidates.find((c) => c.targetID === 103)?.reason).toContain("unpublished"); + expect(result.candidates.find((c) => c.targetID === 105)?.reason).toContain("content posts #900"); + }); + + it("deletes children before parents and removes only the deleted mappings", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await run(); + + expect(fixture.deleted).toEqual( + expect.arrayContaining(["page:112", "page:111", "content:102", "container:131", "model:121", "asset:151", "gallery:161"]) + ); + expect(fixture.deleted.indexOf("page:112")).toBeLessThan(fixture.deleted.indexOf("page:111")); + expect(fixture.deleted).not.toContain("content:104"); + expect(fixture.deleted).not.toContain("template:141"); + expect(deleteUrlRedirections).toHaveBeenCalledWith(TGT, [171]); + + expect(readMapping(fixture, `${LOCALE}/item`).map((r) => r.targetContentID)).toEqual([101, 103, 104, 105]); + expect(readMapping(fixture, `${LOCALE}/page`).map((r) => r.targetPageID)).toEqual([110, 113]); + expect(readMapping(fixture, "models").map((r) => r.targetID)).toEqual([120, 122]); + expect(readMapping(fixture, "urlredirections").map((r) => r.targetUrlRedirectionID)).toEqual([170]); + + expect(result.successful).toBe(8); + expect(result.failed).toBe(0); + expect(result.warningDetails!.map((w) => w.name).sort()).toEqual(["InUse", "Old", "page-113", "posts #104", "posts #105"].sort()); + }); + + it("deletes a target edited since the sync only with --overwrite", async () => { + const fixture = buildFixture(); + configure(fixture, { overwrite: true }); + const result = await run(); + expect(actions(result)["content:104"]).toBe("delete"); + expect(fixture.deleted).toContain("content:104"); + }); + + it("writes nothing under --preflight and reports the plan", async () => { + const fixture = buildFixture(); + configure(fixture, { preflight: true }); + const before = readMapping(fixture, `${LOCALE}/item`); + await run(); + + expect(fixture.deleted).toEqual([]); + expect(deleteUrlRedirections).not.toHaveBeenCalled(); + expect(readMapping(fixture, `${LOCALE}/item`)).toEqual(before); + const totals = preflightReport.getTotals(); + expect(totals.delete).toBe(8); + expect(totals.conflict).toBe(1); + expect(preflightReport.getEntries().find((e) => e.name === "posts #102")).toMatchObject({ + phase: "Delete Content", + action: "delete", + locale: LOCALE, + }); + }); + + it("counts a target that disappeared before its delete (e.g. with its page) as done", async () => { + const fixture = buildFixture(); + configure(fixture); + fixture.api.pageMethods.deletePage.mockImplementation(async (id: number) => { + fixture.deleted.push(`page:${id}`); + if (id === 111) fixture.deleted.push("content:102"); // page delete took its zone content along + return [2]; + }); + const result = await run(); + expect(fixture.api.contentMethods.deleteContent).not.toHaveBeenCalledWith(102, expect.anything(), expect.anything(), expect.anything(), expect.anything()); + expect(result.failed).toBe(0); + expect(readMapping(fixture, `${LOCALE}/item`).map((r) => r.targetContentID)).not.toContain(102); + }); + + it("reports a failed delete and keeps its mapping", async () => { + const fixture = buildFixture(); + configure(fixture); + fixture.api.modelMethods.deleteModel.mockRejectedValue(new Error("model in use")); + const result = await run(); + expect(result.failed).toBe(1); + expect(result.status).toBe("error"); + expect(result.failureDetails![0]).toMatchObject({ name: "Gone", type: "model" }); + expect(readMapping(fixture, "models").map((r) => r.targetID)).toContain(121); + }); + + it("does not delete anything of a kind whose source list is empty", async () => { + const fixture = buildFixture(); + configure(fixture); + fs.rmSync(path.join(fixture.root, SRC, "models"), { recursive: true }); + const result = await run(); + expect(actions(result)["model:121"]).toBe("keep"); + expect(fixture.deleted).not.toContain("model:121"); + }); + + it("only considers the kinds in --elements", async () => { + const fixture = buildFixture(); + configure(fixture); + const result = await runSyncDeletes({ sourceGuid: SRC, targetGuid: TGT, locales: [LOCALE], elements: ["Content"] }); + expect(new Set(result.candidates.map((c) => c.kind))).toEqual(new Set(["content"])); + }); +}); + +describe("mass-delete guard", () => { + function massFixture(): Fixture { + const fixture = buildFixture(); + const records = Array.from({ length: 12 }, (_, i) => ({ + sourceContentID: 200 + i, + targetContentID: 300 + i, + sourceVersionID: 1, + targetVersionID: 1, + })); + records.forEach((r) => + writeJson(path.join(fixture.root, TGT, LOCALE, "item", `${r.targetContentID}.json`), { + contentID: r.targetContentID, + properties: { state: 2, versionID: 1, referenceName: "posts" }, + fields: {}, + }) + ); + writeJson(path.join(fixture.root, "mappings", `${SRC}-${TGT}`, LOCALE, "item", "mappings.json"), [ + { sourceContentID: 1, targetContentID: 101, sourceVersionID: 1, targetVersionID: 1 }, + ...records, + ]); + return fixture; + } + + it("refuses to delete anything when one kind would lose most of its mapped entities", async () => { + const fixture = massFixture(); + configure(fixture); + const result = await run(); + expect(result.massDeleteRefused).toBe(true); + expect(result.failed).toBe(1); + expect(fixture.deleted).toEqual([]); + expect(result.failureDetails![0].error).toContain("--allowMassDeletes"); + }); + + it("goes ahead with --allowMassDeletes", async () => { + const fixture = massFixture(); + configure(fixture, { allowMassDeletes: true }); + const result = await run(); + expect(result.massDeleteRefused).toBe(false); + expect(fixture.deleted.filter((d) => d.startsWith("content:3"))).toHaveLength(12); + }); +}); diff --git a/src/lib/preflight/preflight-report.ts b/src/lib/preflight/preflight-report.ts index 00dba94..3a3e3a7 100644 --- a/src/lib/preflight/preflight-report.ts +++ b/src/lib/preflight/preflight-report.ts @@ -18,7 +18,7 @@ import { state } from "../../core/state"; * the target model (e.g. a missing required field), so a real run skips it and reports a failure. * It is a warning in the plan: not a write, and — unlike "conflict" — it doesn't set the exit code. */ -export type PreflightAction = "create" | "update" | "skip" | "conflict" | "invalid"; +export type PreflightAction = "create" | "update" | "delete" | "skip" | "conflict" | "invalid"; export interface PreflightEntry { /** Phase / element type, e.g. "Models", "Containers", "Content". */ @@ -37,17 +37,19 @@ export interface PreflightPhaseSummary { phase: string; create: number; update: number; + delete: number; skip: number; conflict: number; invalid: number; entries: PreflightEntry[]; } -const ACTION_ORDER: PreflightAction[] = ["create", "update", "skip", "conflict", "invalid"]; +const ACTION_ORDER: PreflightAction[] = ["create", "update", "delete", "skip", "conflict", "invalid"]; const ACTION_COLOR: Record string> = { create: ansiColors.green, update: ansiColors.cyan, + delete: ansiColors.magenta, skip: ansiColors.gray, conflict: ansiColors.red, invalid: ansiColors.yellow, @@ -56,6 +58,7 @@ const ACTION_COLOR: Record string> = { const ACTION_GLYPH: Record = { create: "+", update: "~", + delete: "-", skip: "·", conflict: "✗", invalid: "!", @@ -112,7 +115,7 @@ class PreflightReport { for (const entry of this.entries) { let summary = byPhase.get(entry.phase); if (!summary) { - summary = { phase: entry.phase, create: 0, update: 0, skip: 0, conflict: 0, invalid: 0, entries: [] }; + summary = { phase: entry.phase, create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0, entries: [] }; byPhase.set(entry.phase, summary); } summary[entry.action]++; @@ -123,7 +126,7 @@ class PreflightReport { /** Aggregate totals across all phases. */ getTotals(): Record { - const totals: Record = { create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }; + const totals: Record = { create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }; for (const entry of this.entries) { totals[entry.action]++; } @@ -148,6 +151,7 @@ class PreflightReport { phase: string; create: number; update: number; + delete: number; skip: number; conflict: number; invalid: number; @@ -157,10 +161,11 @@ class PreflightReport { return { totals: this.getTotals(), hasConflicts: this.hasConflicts(), - phases: this.getPhaseSummaries().map(({ phase, create, update, skip, conflict, invalid }) => ({ + phases: this.getPhaseSummaries().map(({ phase, create, update, delete: deletes, skip, conflict, invalid }) => ({ phase, create, update, + delete: deletes, skip, conflict, invalid, @@ -189,7 +194,7 @@ class PreflightReport { const phases = this.getPhaseSummaries(); if (phases.length === 0) { - lines.push(ansiColors.gray("\nNothing to do — no creates, updates, skips, or conflicts detected.\n")); + lines.push(ansiColors.gray("\nNothing to do — no creates, updates, deletes, skips, or conflicts detected.\n")); lines.push(...this.renderWarnings()); return lines.join("\n"); } @@ -199,6 +204,7 @@ class PreflightReport { `\n${ansiColors.bold(phase.phase)}: ` + ACTION_COLOR.create(`${phase.create} create, `) + ACTION_COLOR.update(`${phase.update} update, `) + + (phase.delete > 0 ? ACTION_COLOR.delete(`${phase.delete} delete, `) : "") + ACTION_COLOR.skip(`${phase.skip} skip, `) + ACTION_COLOR.conflict(`${phase.conflict} conflict`) + (phase.invalid > 0 ? ACTION_COLOR.invalid(`, ${phase.invalid} invalid`) : ""); @@ -222,6 +228,7 @@ class PreflightReport { ansiColors.bold("TOTAL: ") + ACTION_COLOR.create(`${totals.create} create, `) + ACTION_COLOR.update(`${totals.update} update, `) + + (totals.delete > 0 ? ACTION_COLOR.delete(`${totals.delete} delete, `) : "") + ACTION_COLOR.skip(`${totals.skip} skip, `) + ACTION_COLOR.conflict(`${totals.conflict} conflict`) + (totals.invalid > 0 ? ACTION_COLOR.invalid(`, ${totals.invalid} invalid`) : "") diff --git a/src/lib/preflight/tests/preflight-report.test.ts b/src/lib/preflight/tests/preflight-report.test.ts index ab10e6b..d702440 100644 --- a/src/lib/preflight/tests/preflight-report.test.ts +++ b/src/lib/preflight/tests/preflight-report.test.ts @@ -121,7 +121,7 @@ describe("hasConflicts", () => { describe("getTotals", () => { it("returns all-zeros totals when there are no entries", () => { setState({ preflight: true }); - expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); it("correctly counts each action type", () => { @@ -133,14 +133,14 @@ describe("getTotals", () => { preflightReport.record({ phase: "Content", action: "skip", name: "c3" }); preflightReport.record({ phase: "Pages", action: "conflict", name: "p1" }); - expect(preflightReport.getTotals()).toEqual({ create: 2, update: 1, skip: 2, conflict: 1, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 2, update: 1, delete: 0, skip: 2, conflict: 1, invalid: 0 }); }); it("counts only the actions that appear", () => { setState({ preflight: true }); preflightReport.record({ phase: "Models", action: "update", name: "m1" }); - expect(preflightReport.getTotals()).toEqual({ create: 0, update: 1, skip: 0, conflict: 0, invalid: 0 }); + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 1, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); }); @@ -292,7 +292,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); - expect(json.totals).toEqual({ create: 1, update: 1, skip: 1, conflict: 1, invalid: 0 }); + expect(json.totals).toEqual({ create: 1, update: 1, delete: 0, skip: 1, conflict: 1, invalid: 0 }); expect(json.hasConflicts).toBe(true); }); @@ -306,7 +306,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); expect(json.phases.map((p) => p.phase)).toEqual(["Models", "Containers"]); - expect(json.phases[0]).toEqual({ phase: "Models", create: 2, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(json.phases[0]).toEqual({ phase: "Models", create: 2, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); }); it("carries every entry through, with locale and detail intact", () => { @@ -326,7 +326,7 @@ describe("toJSON", () => { const json = preflightReport.toJSON(); - expect(json.totals).toEqual({ create: 0, update: 0, skip: 0, conflict: 0, invalid: 0 }); + expect(json.totals).toEqual({ create: 0, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 0 }); expect(json.hasConflicts).toBe(false); expect(json.phases).toEqual([]); expect(json.entries).toEqual([]); @@ -392,10 +392,10 @@ describe("invalid action (PROD-2542)", () => { detail: "would create but Missing required field(s) on model PromoBanner: promoTitle (Text)", }); - expect(preflightReport.getTotals()).toEqual({ create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }); + expect(preflightReport.getTotals()).toEqual({ create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }); expect(preflightReport.hasConflicts()).toBe(false); expect(preflightReport.toJSON().phases).toEqual([ - { phase: "Content", create: 1, update: 0, skip: 0, conflict: 0, invalid: 1 }, + { phase: "Content", create: 1, update: 0, delete: 0, skip: 0, conflict: 0, invalid: 1 }, ]); }); @@ -459,3 +459,23 @@ describe("recordWarning", () => { expect(preflightReport.getWarnings()).toEqual([]); }); }); + +// ─── delete action (--syncDeletes) ──────────────────────────────────────────── + +describe("delete action (--syncDeletes)", () => { + beforeEach(() => { + setState({ preflight: true }); + preflightReport.reset(); + }); + + it("is counted on its own and rendered in the phase header and totals", () => { + preflightReport.record({ phase: "Delete Content", action: "delete", name: "posts #12", locale: "en-us", detail: "deleted in the source" }); + preflightReport.record({ phase: "Delete Content", action: "conflict", name: "posts #13", locale: "en-us" }); + + expect(preflightReport.getTotals()).toEqual({ create: 0, update: 0, delete: 1, skip: 0, conflict: 1, invalid: 0 }); + // eslint-disable-next-line no-control-regex + const table = preflightReport.renderTable().replace(/\x1b\[[0-9;]*m/g, ""); + expect(table).toContain("Delete Content: 0 create, 0 update, 1 delete, 0 skip, 1 conflict"); + expect(table).toContain("- [en-us] posts #12 — deleted in the source"); + }); +}); diff --git a/src/lib/pushers/orchestrate-pushers.ts b/src/lib/pushers/orchestrate-pushers.ts index 57b49a4..0b66ff1 100644 --- a/src/lib/pushers/orchestrate-pushers.ts +++ b/src/lib/pushers/orchestrate-pushers.ts @@ -226,6 +226,17 @@ export class Pushers { (state.models && state.models.trim().length > 0) || (state.modelsWithDeps && state.modelsWithDeps.trim().length > 0); + // --syncDeletes decides "deleted in the source" from what is absent in the source. A scoped run + // narrows the source to part of the instance, so everything outside the scope would look + // deleted. Refuse the combination before anything is written. + if (state.syncDeletes && (pageSelectors.length > 0 || containerSelectors.length > 0 || hasModelFilter)) { + throw new Error( + "Sync deletes validation failed. --syncDeletes cannot be combined with --pages, --containers, --models " + + "or --models-with-deps — a scoped run cannot tell a deleted entity from one outside the scope. " + + "Run the deletes as a separate, unscoped sync (use --elements to limit the entity types)." + ); + } + if (pageSelectors.length > 0 && containerSelectors.length > 0) { throw new Error( "Page validation failed. --pages cannot be combined with --containers — each defines a " + @@ -386,6 +397,26 @@ export class Pushers { } } + // --syncDeletes runs last: every push has repointed what it could, so the targets that are + // still only reachable from a deleted source can go, dependents first. + if (state.syncDeletes) { + const { runSyncDeletes } = await import("../deleters/sync-deletes"); + const deletes = await runSyncDeletes({ sourceGuid, targetGuid, locales, elements }); + totalSuccess += deletes.successful; + totalFailures += deletes.failed; + totalSkipped += deletes.skipped; + failureDetails.push(...(deletes.failureDetails || [])); + warningDetails.push(...(deletes.warningDetails || [])); + phases.push({ + operation: "syncDeletes", + description: "Delete entities deleted in the source", + successful: deletes.successful, + failed: deletes.failed, + skipped: deletes.skipped, + status: deletes.status === "error" ? "error" : "success", + }); + } + return { totalSuccess, totalFailures, diff --git a/src/lib/pushers/url-redirection-api.ts b/src/lib/pushers/url-redirection-api.ts index 329aa3e..822ba9a 100644 --- a/src/lib/pushers/url-redirection-api.ts +++ b/src/lib/pushers/url-redirection-api.ts @@ -1,6 +1,10 @@ import { Auth } from "../../core/auth"; import { state } from "../../core/state"; -import { UrlRedirectionSavePayload, UrlRedirectionSaveResult } from "../../types/urlRedirection"; +import { + UrlRedirectionDeleteResult, + UrlRedirectionSavePayload, + UrlRedirectionSaveResult, +} from "../../types/urlRedirection"; /** The endpoint processes at most 250 redirections per request. */ export const MAX_URL_REDIRECTION_BATCH_SIZE = 250; @@ -57,3 +61,48 @@ export async function saveUrlRedirections( skipped: result?.skipped ?? [], }; } + +/** + * Delete a batch of URL redirections via the Management API (--syncDeletes): + * DELETE /api/v1/instance/{guid}/url-redirections?ids=1,2,3 + * + * Like the save endpoint this is not in @agility/management-sdk yet. IDs that no longer match an + * active redirection come back in `notFound` rather than failing the request. + */ +export async function deleteUrlRedirections(guid: string, ids: number[]): Promise { + if (ids.length > MAX_URL_REDIRECTION_BATCH_SIZE) { + throw new Error( + `Cannot delete more than ${MAX_URL_REDIRECTION_BATCH_SIZE} redirections at a time (got ${ids.length}).` + ); + } + + const auth = new Auth(); + const baseUrl = state.baseUrl || auth.determineBaseUrl(guid); + const token = await auth.getToken(); + const url = `${baseUrl}/api/v1/instance/${guid}/url-redirections?ids=${ids.join(",")}`; + + const response = await fetch(url, { + method: "DELETE", + headers: { + Authorization: `Bearer ${token}`, + "Cache-Control": "no-cache", + "User-Agent": "agility-cli-fetch/1.0", + }, + }); + + if (!response.ok) { + let body = ""; + try { + body = await response.text(); + } catch { + // response body is best-effort error context + } + throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}${body ? ` — ${body}` : ""}`); + } + + const result = (await response.json()) as any; + return { + deleted: result?.deleted ?? result?.Deleted ?? [], + notFound: result?.notFound ?? result?.NotFound ?? [], + }; +} diff --git a/src/tests/sync/helpers/fake-instance.ts b/src/tests/sync/helpers/fake-instance.ts index 4c89476..9c314c5 100644 --- a/src/tests/sync/helpers/fake-instance.ts +++ b/src/tests/sync/helpers/fake-instance.ts @@ -218,6 +218,11 @@ export class FakeInstance { // ─── the mgmtApi.ApiClient surface ───────────────────────────────────────── modelMethods = { + deleteModel: async (id: number, guid: string) => { + this.record("modelMethods.deleteModel", guid, { id }); + this.instance(guid).models.delete(id); + return "Model deleted successfully."; + }, saveModel: async (model: any, guid: string) => { this.record("modelMethods.saveModel", guid, model); const store = this.instance(guid); @@ -243,6 +248,11 @@ export class FakeInstance { }; containerMethods = { + deleteContainer: async (id: number, guid: string) => { + this.record("containerMethods.deleteContainer", guid, { id }); + this.instance(guid).containers.delete(id); + return "Container deleted successfully."; + }, saveContainer: async (container: any, guid: string) => { this.record("containerMethods.saveContainer", guid, container); const store = this.instance(guid); @@ -275,6 +285,11 @@ export class FakeInstance { private lastPageSaveIds: number[] = []; contentMethods = { + deleteContent: async (id: number, guid: string, locale: string) => { + this.record("contentMethods.deleteContent", guid, { id }, locale); + this.instance(guid).content.delete(`${locale}:${id}`); + return [3000 + this.callsTo("contentMethods.deleteContent").length]; + }, saveContentItems: async (items: any[], guid: string, locale: string) => { this.record("contentMethods.saveContentItems", guid, items, locale); const store = this.instance(guid); @@ -309,6 +324,11 @@ export class FakeInstance { }; assetMethods = { + deleteGallery: async (guid: string, id: number) => { + this.record("assetMethods.deleteGallery", guid, { id }); + this.instance(guid).galleries.delete(id); + return "Gallery deleted successfully."; + }, getMediaList: async (pageSize: number, recordOffset: number, guid: string) => { this.record("assetMethods.getMediaList", guid, { pageSize, recordOffset }); // One page only: returning the full list on every call would loop forever if the @@ -350,6 +370,9 @@ export class FakeInstance { }, deleteFile: async (mediaID: number, guid: string) => { this.record("assetMethods.deleteFile", guid, { mediaID }); + const assets = this.instance(guid).assets; + const index = assets.findIndex((a: any) => a.mediaID === mediaID); + if (index >= 0) assets.splice(index, 1); return true; }, deleteFolder: async (originKey: string, guid: string) => { @@ -359,6 +382,16 @@ export class FakeInstance { }; pageMethods = { + deletePage: async (id: number, guid: string, locale: string) => { + this.record("pageMethods.deletePage", guid, { id }, locale); + this.instance(guid).pages.delete(`${locale}:${id}`); + return [4000 + this.callsTo("pageMethods.deletePage").length]; + }, + deletePageTemplate: async (guid: string, locale: string, id: number) => { + this.record("pageMethods.deletePageTemplate", guid, { id }, locale); + this.instance(guid).templates.delete(id); + return "Template deleted successfully."; + }, getSitemap: async (guid: string, locale: string) => { this.record("pageMethods.getSitemap", guid, undefined, locale); return this.instance(guid).sitemap; diff --git a/src/tests/sync/helpers/run-sync.ts b/src/tests/sync/helpers/run-sync.ts index 47ab159..7a7783d 100644 --- a/src/tests/sync/helpers/run-sync.ts +++ b/src/tests/sync/helpers/run-sync.ts @@ -64,6 +64,8 @@ export interface RunSyncOptions { fixture: SyncFixture; /** Pre-seed the fake's target instance, for entities that exist server-side already. */ seedTarget?: (fake: FakeInstance, targetGuid: string) => void; + /** Extra CLI state for every run (e.g. { syncDeletes: true }). Can be changed between runs. */ + stateOverrides?: Record; } export interface SyncRun { @@ -84,6 +86,8 @@ export class SyncHarness { private readonly elements: string; private readonly overwrite: boolean; private readonly preflight: boolean; + /** Extra CLI state applied on every run; reassign between runs to change flags for run 2. */ + stateOverrides: Record; constructor(opts: RunSyncOptions) { this.sourceGuid = opts.sourceGuid ?? "src-1"; @@ -92,6 +96,7 @@ export class SyncHarness { this.elements = opts.elements ?? "Models,Containers,Content"; this.overwrite = opts.overwrite ?? false; this.preflight = opts.preflight ?? false; + this.stateOverrides = opts.stateOverrides ?? {}; this.rootPath = fs.mkdtempSync(path.join(os.tmpdir(), "agility-sync-harness-")); this.fake = new FakeInstance({ locales: this.locales }); @@ -130,6 +135,7 @@ export class SyncHarness { elements: this.elements, overwrite: this.overwrite, preflight: this.preflight, + ...this.stateOverrides, }); state.cachedApiClient = this.fake.asApiClient(); state.availableLocales = [...this.locales]; diff --git a/src/tests/sync/sync-deletes.test.ts b/src/tests/sync/sync-deletes.test.ts new file mode 100644 index 0000000..23472e8 --- /dev/null +++ b/src/tests/sync/sync-deletes.test.ts @@ -0,0 +1,179 @@ +/** + * --syncDeletes end to end: sync, delete something in the source, re-sync with the flag, and the + * target loses exactly that — through the real orchestrator, mappings and all. + */ +import * as fs from "fs"; +import * as path from "path"; +import { SyncHarness } from "./helpers/run-sync"; +import { installNetworkStubs, InstalledNetworkStubs } from "./helpers/network-stubs"; + +jest.mock("lib/pushers/batch-polling", () => ({ + pollBatchUntilComplete: jest.fn(), + extractContentBatchResults: jest.fn(), + extractPageBatchResults: jest.fn(), +})); + +const POST_MODEL = { + id: 10, + displayName: "Post", + referenceName: "Post", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Title", label: "Title", required: true }], +}; + +const AUTHOR_MODEL = { + id: 11, + displayName: "Author", + referenceName: "Author", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Name", label: "Name", required: true }], +}; + +const NEWS_MODEL = { + id: 12, + displayName: "News", + referenceName: "News", + lastModifiedDate: "2025-01-01T00:00:00.000", + contentDefinitionTypeName: "Content List", + fields: [{ type: "Text", name: "Headline", label: "Headline", required: true }], +}; + +const POSTS_CONTAINER = { + contentViewID: 200, + referenceName: "Posts", + title: "Posts", + contentDefinitionID: 10, + contentDefinitionName: "Post", + lastModifiedDate: "01/01/2025 12:00AM", + isShared: false, +}; + +const AUTHORS_CONTAINER = { + contentViewID: 201, + referenceName: "Authors", + title: "Authors", + contentDefinitionID: 11, + contentDefinitionName: "Author", + lastModifiedDate: "01/01/2025 12:00AM", + isShared: false, +}; + +function fixture() { + return { + source: { + "models/10.json": POST_MODEL, + "models/11.json": AUTHOR_MODEL, + "models/12.json": NEWS_MODEL, + "containers/200.json": POSTS_CONTAINER, + "containers/201.json": AUTHORS_CONTAINER, + }, + }; +} + +describe("--syncDeletes through the orchestrator", () => { + let harness: SyncHarness; + let net: InstalledNetworkStubs; + + beforeEach(() => { + net = installNetworkStubs(); + jest.spyOn(console, "log").mockImplementation(() => {}); + jest.spyOn(console, "warn").mockImplementation(() => {}); + jest.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + harness?.cleanup(); + net.restore(); + jest.restoreAllMocks(); + }); + + function deleteFromSource(rel: string) { + fs.rmSync(path.join(harness.rootPath, harness.sourceGuid, rel)); + } + + it("deletes the target counterparts of source deletes and drops their mappings", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const authorTarget = harness.readMappings("models")!.find((m) => m.sourceID === 11)!.targetID; + const authorsContainerTarget = harness.readMappings("containers")!.find((m) => m.sourceContentViewID === 201)!.targetContentViewID; + + deleteFromSource("models/11.json"); + deleteFromSource("containers/201.json"); + harness.stateOverrides = { syncDeletes: true }; + const run2 = await harness.resync(); + + const target = harness.fake.instance(harness.targetGuid); + expect(target.models.has(authorTarget)).toBe(false); + expect(target.containers.has(authorsContainerTarget)).toBe(false); + expect(target.models.size).toBe(2); + // Container before its model. + const order = harness.fake.calls.map((c) => c.method); + expect(order.indexOf("containerMethods.deleteContainer")).toBeLessThan(order.indexOf("modelMethods.deleteModel")); + + expect(harness.readMappings("models")!.map((m) => m.sourceID).sort()).toEqual([10, 12]); + expect(harness.readMappings("containers")!.map((m) => m.sourceContentViewID)).toEqual([200]); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toMatchObject({ successful: 2, failed: 0 }); + }); + + it("leaves source deletes alone without the flag", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + + deleteFromSource("models/12.json"); + const run2 = await harness.resync(); + + expect(harness.fake.instance(harness.targetGuid).models.size).toBe(3); + expect(harness.fake.callsTo("modelMethods.deleteModel")).toEqual([]); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toBeUndefined(); + }); + + it("keeps a model that a target-only container still uses", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const newsTarget = harness.readMappings("models")!.find((m) => m.sourceID === 12)!.targetID; + // Someone built a list on the target from the News model. + harness.fake.seedContainer(harness.targetGuid, { + contentViewID: 8999, + referenceName: "TargetNews", + contentDefinitionID: newsTarget, + lastModifiedDate: "01/01/2025 12:00AM", + }); + + deleteFromSource("models/12.json"); + harness.stateOverrides = { syncDeletes: true }; + const run2 = await harness.resync(); + + expect(harness.fake.instance(harness.targetGuid).models.has(newsTarget)).toBe(true); + expect(run2.phases.find((p) => p.operation === "syncDeletes")).toMatchObject({ successful: 0, skipped: 1 }); + const warning = run2.results[0].warningDetails.find((w: any) => w.name === "News"); + expect(warning?.error).toContain("container TargetNews"); + }); + + it("previews deletes under --preflight without writing", async () => { + harness = new SyncHarness({ elements: "Models,Containers", fixture: fixture() }); + await harness.run(); + const before = harness.readAllMappings(); + + deleteFromSource("models/12.json"); + harness.stateOverrides = { syncDeletes: true, preflight: true }; + await harness.resync(); + + expect(harness.fake.callsTo("modelMethods.deleteModel")).toEqual([]); + expect(harness.readAllMappings()).toEqual(before); + }); + + it("refuses --syncDeletes on a scoped run before writing anything", async () => { + harness = new SyncHarness({ + elements: "Models,Containers", + fixture: fixture(), + stateOverrides: { syncDeletes: true, models: "Post" }, + }); + + const run = await harness.run(); + + expect(run.results[0].failed[0].error).toContain("Sync deletes validation failed"); + expect(harness.fake.calls.filter((c) => !c.method.startsWith("get"))).toEqual([]); + }); +}); diff --git a/src/types/urlRedirection.ts b/src/types/urlRedirection.ts index 688e4d3..c447cdf 100644 --- a/src/types/urlRedirection.ts +++ b/src/types/urlRedirection.ts @@ -55,3 +55,11 @@ export interface UrlRedirectionSaveResult { updated: UrlRedirectionItemResult[]; skipped: UrlRedirectionItemResult[]; } + +/** Response of DELETE /api/v1/instance/{guid}/url-redirections?ids=... */ +export interface UrlRedirectionDeleteResult { + /** IDs of the redirections that were deleted. */ + deleted: number[]; + /** IDs that did not match an active redirection. */ + notFound: number[]; +} From d8662ccd8aada844dd82da50e37f0710b4a984ae Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 30 Oct 2026 12:13:57 -0400 Subject: [PATCH 2/3] Delete URL redirections through the SDK's urlRedirectionMethods --syncDeletes now deletes redirections with apiClient.urlRedirectionMethods.deleteUrlRedirections (management SDK 0.2.0-beta.3) instead of its own fetch call, and chunks by the SDK's MAX_URL_REDIRECTIONS_PER_REQUEST. The CLI's hand-rolled delete helper and its result type are removed. Bumps @agility/management-sdk to ^0.2.0-beta.3. That version is not on npm yet, so the lockfile is regenerated once it is published. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- src/lib/deleters/sync-deletes.ts | 12 +++-- src/lib/deleters/tests/sync-deletes.test.ts | 17 ++++--- src/lib/pushers/url-redirection-api.ts | 51 +-------------------- src/tests/sync/helpers/fake-instance.ts | 9 ++++ src/tests/sync/sync-deletes.test.ts | 38 +++++++++++++++ src/types/urlRedirection.ts | 8 ---- 7 files changed, 65 insertions(+), 72 deletions(-) diff --git a/package.json b/package.json index 0aceaf4..23a63fd 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "dependencies": { "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.38", + "@agility/management-sdk": "^0.2.0-beta.3", "ansi-colors": "^4.1.3", "axios": "^0.27.2", "cli-progress": "^3.11.2", diff --git a/src/lib/deleters/sync-deletes.ts b/src/lib/deleters/sync-deletes.ts index 3dfcc00..eb40815 100644 --- a/src/lib/deleters/sync-deletes.ts +++ b/src/lib/deleters/sync-deletes.ts @@ -27,7 +27,7 @@ import { state, getApiClient, getLoggerForGuid } from "../../core/state"; import { fileOperations } from "../../core/fileOperations"; import { GuidDataLoader, GuidEntities } from "../pushers/guid-data-loader"; import { preflightReport } from "../preflight/preflight-report"; -import { deleteUrlRedirections, MAX_URL_REDIRECTION_BATCH_SIZE } from "../pushers/url-redirection-api"; +import { MAX_URL_REDIRECTIONS_PER_REQUEST } from "@agility/management-sdk"; import { FailureDetail, FailureDetailType, PusherResult } from "../../types/sourceData"; import { CandidateFacts, @@ -821,11 +821,15 @@ async function executeGroup(candidates: DeleteCandidate[], opts: SyncDeletesOpti const kind = toDelete[0].kind; if (kind === "urlRedirection") { - for (let i = 0; i < toDelete.length; i += MAX_URL_REDIRECTION_BATCH_SIZE) { - const chunk = toDelete.slice(i, i + MAX_URL_REDIRECTION_BATCH_SIZE); + const api = getApiClient(); + for (let i = 0; i < toDelete.length; i += MAX_URL_REDIRECTIONS_PER_REQUEST) { + const chunk = toDelete.slice(i, i + MAX_URL_REDIRECTIONS_PER_REQUEST); try { // notFound means it is already gone, which is the outcome we wanted. - await deleteUrlRedirections(state.targetGuid, chunk.map((c) => c.targetID)); + await api.urlRedirectionMethods.deleteUrlRedirections( + state.targetGuid, + chunk.map((c) => c.targetID) + ); } catch (error: any) { chunk.forEach((c) => failures.set(c, errorMessage(error))); } diff --git a/src/lib/deleters/tests/sync-deletes.test.ts b/src/lib/deleters/tests/sync-deletes.test.ts index 3a657a6..e7d2cc2 100644 --- a/src/lib/deleters/tests/sync-deletes.test.ts +++ b/src/lib/deleters/tests/sync-deletes.test.ts @@ -2,15 +2,9 @@ import * as fs from "fs"; import * as os from "os"; import * as path from "path"; -jest.mock("../../pushers/url-redirection-api", () => ({ - MAX_URL_REDIRECTION_BATCH_SIZE: 250, - deleteUrlRedirections: jest.fn(async (_guid: string, ids: number[]) => ({ deleted: ids, notFound: [] })), -})); - import { state, resetState, setState } from "../../../core/state"; import { preflightReport } from "../../preflight/preflight-report"; import { runSyncDeletes } from "../sync-deletes"; -import { deleteUrlRedirections } from "../../pushers/url-redirection-api"; const SRC = "src-1"; const TGT = "tgt-1"; @@ -139,6 +133,12 @@ function buildFixture(): Fixture { return "ok"; }), }, + urlRedirectionMethods: { + deleteUrlRedirections: jest.fn(async (_guid: string, ids: number[]) => { + ids.forEach((id) => deleted.push(`urlRedirection:${id}`)); + return { deleted: ids, notFound: [] }; + }), + }, batchMethods: { getBatch: jest.fn(async () => ({ batchState: 3, items: [] })) }, containerMethods: { deleteContainer: jest.fn(async (id: number) => { @@ -193,7 +193,6 @@ function actions(result: Awaited>) { let logSpy: jest.SpyInstance; beforeEach(() => { logSpy = jest.spyOn(console, "log").mockImplementation(() => undefined); - (deleteUrlRedirections as jest.Mock).mockClear(); }); afterEach(() => logSpy.mockRestore()); @@ -234,7 +233,7 @@ describe("runSyncDeletes", () => { expect(fixture.deleted.indexOf("page:112")).toBeLessThan(fixture.deleted.indexOf("page:111")); expect(fixture.deleted).not.toContain("content:104"); expect(fixture.deleted).not.toContain("template:141"); - expect(deleteUrlRedirections).toHaveBeenCalledWith(TGT, [171]); + expect(fixture.api.urlRedirectionMethods.deleteUrlRedirections).toHaveBeenCalledWith(TGT, [171]); expect(readMapping(fixture, `${LOCALE}/item`).map((r) => r.targetContentID)).toEqual([101, 103, 104, 105]); expect(readMapping(fixture, `${LOCALE}/page`).map((r) => r.targetPageID)).toEqual([110, 113]); @@ -261,7 +260,7 @@ describe("runSyncDeletes", () => { await run(); expect(fixture.deleted).toEqual([]); - expect(deleteUrlRedirections).not.toHaveBeenCalled(); + expect(fixture.api.urlRedirectionMethods.deleteUrlRedirections).not.toHaveBeenCalled(); expect(readMapping(fixture, `${LOCALE}/item`)).toEqual(before); const totals = preflightReport.getTotals(); expect(totals.delete).toBe(8); diff --git a/src/lib/pushers/url-redirection-api.ts b/src/lib/pushers/url-redirection-api.ts index 822ba9a..329aa3e 100644 --- a/src/lib/pushers/url-redirection-api.ts +++ b/src/lib/pushers/url-redirection-api.ts @@ -1,10 +1,6 @@ import { Auth } from "../../core/auth"; import { state } from "../../core/state"; -import { - UrlRedirectionDeleteResult, - UrlRedirectionSavePayload, - UrlRedirectionSaveResult, -} from "../../types/urlRedirection"; +import { UrlRedirectionSavePayload, UrlRedirectionSaveResult } from "../../types/urlRedirection"; /** The endpoint processes at most 250 redirections per request. */ export const MAX_URL_REDIRECTION_BATCH_SIZE = 250; @@ -61,48 +57,3 @@ export async function saveUrlRedirections( skipped: result?.skipped ?? [], }; } - -/** - * Delete a batch of URL redirections via the Management API (--syncDeletes): - * DELETE /api/v1/instance/{guid}/url-redirections?ids=1,2,3 - * - * Like the save endpoint this is not in @agility/management-sdk yet. IDs that no longer match an - * active redirection come back in `notFound` rather than failing the request. - */ -export async function deleteUrlRedirections(guid: string, ids: number[]): Promise { - if (ids.length > MAX_URL_REDIRECTION_BATCH_SIZE) { - throw new Error( - `Cannot delete more than ${MAX_URL_REDIRECTION_BATCH_SIZE} redirections at a time (got ${ids.length}).` - ); - } - - const auth = new Auth(); - const baseUrl = state.baseUrl || auth.determineBaseUrl(guid); - const token = await auth.getToken(); - const url = `${baseUrl}/api/v1/instance/${guid}/url-redirections?ids=${ids.join(",")}`; - - const response = await fetch(url, { - method: "DELETE", - headers: { - Authorization: `Bearer ${token}`, - "Cache-Control": "no-cache", - "User-Agent": "agility-cli-fetch/1.0", - }, - }); - - if (!response.ok) { - let body = ""; - try { - body = await response.text(); - } catch { - // response body is best-effort error context - } - throw new Error(`HTTP ${response.status}: ${response.statusText} for ${url}${body ? ` — ${body}` : ""}`); - } - - const result = (await response.json()) as any; - return { - deleted: result?.deleted ?? result?.Deleted ?? [], - notFound: result?.notFound ?? result?.NotFound ?? [], - }; -} diff --git a/src/tests/sync/helpers/fake-instance.ts b/src/tests/sync/helpers/fake-instance.ts index 9c314c5..733c0e3 100644 --- a/src/tests/sync/helpers/fake-instance.ts +++ b/src/tests/sync/helpers/fake-instance.ts @@ -434,6 +434,15 @@ export class FakeInstance { }, }; + urlRedirectionMethods = { + deleteUrlRedirections: async (guid: string, ids: number[]) => { + this.record("urlRedirectionMethods.deleteUrlRedirections", guid, { ids }); + const store = this.instance(guid).urlRedirections; + const deleted = ids.filter((id) => store.delete(id)); + return { deleted, notFound: ids.filter((id) => !deleted.includes(id)) }; + }, + }; + instanceMethods = { getLocales: async (guid: string) => { this.record("instanceMethods.getLocales", guid); diff --git a/src/tests/sync/sync-deletes.test.ts b/src/tests/sync/sync-deletes.test.ts index 23472e8..31170fd 100644 --- a/src/tests/sync/sync-deletes.test.ts +++ b/src/tests/sync/sync-deletes.test.ts @@ -13,6 +13,13 @@ jest.mock("lib/pushers/batch-polling", () => ({ extractPageBatchResults: jest.fn(), })); +// Redirection saves build their own URL and call global fetch; the harness wires this to the fake. +// Deletes go through the SDK client (urlRedirectionMethods), which the fake implements directly. +jest.mock("lib/pushers/url-redirection-api", () => ({ + MAX_URL_REDIRECTION_BATCH_SIZE: 250, + saveUrlRedirections: jest.fn(), +})); + const POST_MODEL = { id: 10, displayName: "Post", @@ -164,6 +171,37 @@ describe("--syncDeletes through the orchestrator", () => { expect(harness.readAllMappings()).toEqual(before); }); + it("deletes URL redirections through the SDK's urlRedirectionMethods", async () => { + const redirect = (id: number, origin: string) => ({ id, originUrl: origin, destinationUrl: "/new", statusCode: 301 }); + harness = new SyncHarness({ + elements: "UrlRedirections", + fixture: { + source: { + "en-us/urlredirections/urlredirections.json": { + items: [redirect(1, "/keep"), redirect(2, "/gone")], + lastAccessDate: "2025-01-01T00:00:00.000", + }, + }, + }, + }); + await harness.run(); + const goneTarget = harness.readMappings("urlredirections")!.find((m) => m.sourceUrlRedirectionID === 2)!.targetUrlRedirectionID; + + fs.writeFileSync( + path.join(harness.rootPath, harness.sourceGuid, "en-us", "urlredirections", "urlredirections.json"), + JSON.stringify({ items: [redirect(1, "/keep")], lastAccessDate: "2025-01-02T00:00:00.000" }) + ); + harness.stateOverrides = { syncDeletes: true }; + await harness.resync(); + + expect(harness.fake.callsTo("urlRedirectionMethods.deleteUrlRedirections")).toEqual([ + expect.objectContaining({ guid: harness.targetGuid, payload: { ids: [goneTarget] } }), + ]); + expect(harness.fake.instance(harness.targetGuid).urlRedirections.has(goneTarget)).toBe(false); + expect(harness.readMappings("urlredirections")!.map((m) => m.sourceUrlRedirectionID)).toEqual([1]); + expect(net.fetchCalls).toEqual([]); + }); + it("refuses --syncDeletes on a scoped run before writing anything", async () => { harness = new SyncHarness({ elements: "Models,Containers", diff --git a/src/types/urlRedirection.ts b/src/types/urlRedirection.ts index c447cdf..688e4d3 100644 --- a/src/types/urlRedirection.ts +++ b/src/types/urlRedirection.ts @@ -55,11 +55,3 @@ export interface UrlRedirectionSaveResult { updated: UrlRedirectionItemResult[]; skipped: UrlRedirectionItemResult[]; } - -/** Response of DELETE /api/v1/instance/{guid}/url-redirections?ids=... */ -export interface UrlRedirectionDeleteResult { - /** IDs of the redirections that were deleted. */ - deleted: number[]; - /** IDs that did not match an active redirection. */ - notFound: number[]; -} From f30c2a92efc7786f0e13b07bcdb7f3f5e48cdef3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 30 Oct 2026 13:19:10 -0400 Subject: [PATCH 3/3] Lock @agility/management-sdk 0.2.0-beta.3 Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index e42ebe1..9047abb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "ISC", "dependencies": { "@agility/content-sync": "^1.2.0", - "@agility/management-sdk": "^0.1.38", + "@agility/management-sdk": "^0.2.0-beta.3", "ansi-colors": "^4.1.3", "axios": "^0.27.2", "cli-progress": "^3.11.2", @@ -89,9 +89,9 @@ } }, "node_modules/@agility/management-sdk": { - "version": "0.1.38", - "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.1.38.tgz", - "integrity": "sha512-g6/hNgCjf+uzcJbkPaxeWqwaAid50tMnRW0KYQ4J/fGuJqtcZG0et0X3m76Ev3yUqnkpIO/+C5zMIQMghN3/oQ==", + "version": "0.2.0-beta.3", + "resolved": "https://registry.npmjs.org/@agility/management-sdk/-/management-sdk-0.2.0-beta.3.tgz", + "integrity": "sha512-jBe7eJxOS7HdhfEnuCBBWwPZIGuA4qR+ekwU+wKmMk8HgMVc33BhuQUSajvwSjaZFkrVhTozjYiHNQjz2Y4GVg==", "license": "MIT", "dependencies": { "axios": "^0.27.2"