Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions src/lib/pushers/content-pusher/content-batch-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import {
} from "./util/detect-deleted-linked-list-references";
import { fileOperations } from "core/fileOperations";
import { collectUnresolvedContentReferences } from "./util/has-unresolved-content-references";
import { stripContentReferences } from "./util/strip-content-references";
import { Logs } from "core/logs";
import { state, getFailedContent } from "core/state";

Expand Down Expand Up @@ -49,6 +50,7 @@ export class ContentBatchProcessor {
// that still exist in the source. Used to tell a *deleted* linked-list reference apart
// from a merely "missing" one when clarifying create failures.
private liveSourceContainerRefNames?: Set<string>;
private sourceContentFileNames?: Set<string>;

constructor(config: ContentBatchConfig) {
this.config = {
Expand All @@ -57,6 +59,23 @@ export class ContentBatchProcessor {
};
}

/**
* Whether a source content item no longer exists: no file for it in the source pull, which a pull
* writes for every live item and removes for a deleted one. False when nothing of the source's
* content was pulled for this locale, so a missing pull never reads as "deleted".
*/
private isGoneFromSource(contentID: number): boolean {
if (this.sourceContentFileNames === undefined) {
try {
const sourceFileOps = new fileOperations(this.config.sourceGuid, this.config.locale);
this.sourceContentFileNames = new Set(sourceFileOps.listFilesInFolder("item", ".json"));
} catch {
this.sourceContentFileNames = new Set();
}
}
return this.sourceContentFileNames.size > 0 && !this.sourceContentFileNames.has(`${contentID}.json`);
}

/**
* PROD-2306: Build (and cache) the set of container/linked-list reference names still
* present in the source. When a source linked list is deleted, the pull removes its
Expand Down Expand Up @@ -360,7 +379,7 @@ export class ContentBatchProcessor {
const containerMapper = new ContainerMapper(sourceGuid, targetGuid);
const assetMapper = new AssetMapper(sourceGuid, targetGuid);

for (const contentItem of contentBatch) {
for (let contentItem of contentBatch) {
if (contentItem.properties.definitionName.toLowerCase() === "richtextarea" && contentItem.fields.textblob) {
//if this is a RichText item, we don't need to do the extra processing - just upload it as is

Expand Down Expand Up @@ -460,14 +479,38 @@ export class ContentBatchProcessor {
// NullReferenceException, which reaches the operator as an opaque
// "Object reference not set to an instance of an object." Skip the item here with a
// precise, actionable reason instead of shipping a payload that is guaranteed to fail.
const unresolvedRefs = collectUnresolvedContentReferences(
let unresolvedRefs = collectUnresolvedContentReferences(
contentItem.fields || {},
this.config.referenceMapper,
"",
// PROD-2446: sourceModel carries each field's LinkeContentDropdownValueField/SortIDFieldName
// setting, needed to recognize an unresolved reference living only in a companion field.
sourceModel
);

// A link to an item that is gone from the source (deleted; its copy and mapping removed by
// --syncDeletes) can never resolve, so the guard below would block this item for good.
// The source points at nothing there, so send the item with that link blank, as the source
// has it, and say so. A link to an item that exists but isn't mapped yet still fails.
const goneIDs = Array.from(new Set(unresolvedRefs.map((r) => r.contentID))).filter((id) =>
this.isGoneFromSource(id)
);
if (goneIDs.length > 0) {
const { fields } = stripContentReferences(contentItem.fields || {}, new Set(goneIDs), sourceModel);
contentItem = { ...contentItem, fields };
logger.warning(
`Content item ${contentItem.contentID} (${contentItem.properties?.referenceName}) links to item(s) ` +
`${goneIDs.join(", ")}, which no longer exist in the source; those links are sent blank.`,
contentItem
);
unresolvedRefs = collectUnresolvedContentReferences(
contentItem.fields || {},
this.config.referenceMapper,
"",
sourceModel
);
}

if (unresolvedRefs.length > 0) {
// PROD-2542: when a referenced child failed earlier in this locale (e.g. skipped
// client-side for a missing required field), say so — that's the actual cause.
Expand Down
127 changes: 118 additions & 9 deletions src/lib/pushers/content-pusher/tests/content-batch-processor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -600,9 +600,7 @@ describe("ContentBatchProcessor.prepareContentPayloads — stale container mappi
expect(result.skippedCount).toBe(0);
expect(result.failedItems).toHaveLength(1);
expect(result.failedItems[0].error).toEqual(expect.stringContaining("no longer exists on the target"));
expect(consoleErrorSpy).toHaveBeenCalledWith(
expect.stringContaining("no longer exists on the target")
);
expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining("no longer exists on the target"));
});

it("ships the payload normally when the target container still exists", async () => {
Expand Down Expand Up @@ -633,12 +631,28 @@ describe("ContentBatchProcessor.prepareContentPayloads — trustSameRunMappings
function seed(sourceGuid: string, targetGuid: string): ContentItemMapper {
writeEntityFile(tmpDir, sourceGuid, "models", 1, { id: 1, referenceName: "TestModel", fields: [] });
writeMappingFile(tmpDir, sourceGuid, targetGuid, "models", [
{ sourceGuid, targetGuid, sourceID: 1, targetID: 1, sourceReferenceName: "TestModel", targetReferenceName: "TestModel",
sourceLastModifiedDate: "07/01/2026 12:00PM", targetLastModifiedDate: "07/01/2026 12:00PM" },
{
sourceGuid,
targetGuid,
sourceID: 1,
targetID: 1,
sourceReferenceName: "TestModel",
targetReferenceName: "TestModel",
sourceLastModifiedDate: "07/01/2026 12:00PM",
targetLastModifiedDate: "07/01/2026 12:00PM",
},
]);
writeMappingFile(tmpDir, sourceGuid, targetGuid, "containers", [
{ sourceGuid, targetGuid, sourceContentViewID: 10, targetContentViewID: 999, sourceReferenceName: "ref-1",
targetReferenceName: "ref-1", sourceLastModifiedDate: "07/01/2026 12:00PM", targetLastModifiedDate: "07/01/2026 12:00PM" },
{
sourceGuid,
targetGuid,
sourceContentViewID: 10,
targetContentViewID: 999,
sourceReferenceName: "ref-1",
targetReferenceName: "ref-1",
sourceLastModifiedDate: "07/01/2026 12:00PM",
targetLastModifiedDate: "07/01/2026 12:00PM",
},
]);
writeEntityFile(tmpDir, targetGuid, "containers", 999, { contentViewID: 999, referenceName: "ref-1" });
const referenceMapper = new ContentItemMapper(sourceGuid, targetGuid, "en-us");
Expand Down Expand Up @@ -671,14 +685,109 @@ describe("ContentBatchProcessor.prepareContentPayloads — trustSameRunMappings

it("without the option, treats the missing target as gone (unchanged behavior)", async () => {
const referenceMapper = seed("notrust-src", "notrust-tgt");
const processor = new ContentBatchProcessor(makeConfig({ sourceGuid: "notrust-src", targetGuid: "notrust-tgt", referenceMapper }));
const processor = new ContentBatchProcessor(
makeConfig({ sourceGuid: "notrust-src", targetGuid: "notrust-tgt", referenceMapper })
);

const result = await (processor as any).prepareContentPayloads([item()], "notrust-src", "notrust-tgt", makeLogger());
const result = await (processor as any).prepareContentPayloads(
[item()],
"notrust-src",
"notrust-tgt",
makeLogger()
);

expect(result.prepared[0].payload.contentID).toBe(-1);
});
});

// ─── links to items deleted from the source ───────────────────────────────────

describe("ContentBatchProcessor.prepareContentPayloads — links to items gone from the source", () => {
function seed(sourceGuid: string, targetGuid: string, sourceItemIds: number[]): ContentItemMapper {
writeEntityFile(tmpDir, sourceGuid, "models", 1, { id: 1, referenceName: "TestModel", fields: [] });
writeMappingFile(tmpDir, sourceGuid, targetGuid, "models", [
{
sourceGuid,
targetGuid,
sourceID: 1,
targetID: 1,
sourceReferenceName: "TestModel",
targetReferenceName: "TestModel",
sourceLastModifiedDate: "07/01/2026 12:00PM",
targetLastModifiedDate: "07/01/2026 12:00PM",
},
]);
writeMappingFile(tmpDir, sourceGuid, targetGuid, "containers", [
{
sourceGuid,
targetGuid,
sourceContentViewID: 10,
targetContentViewID: 999,
sourceReferenceName: "ref-1",
targetReferenceName: "ref-1",
sourceLastModifiedDate: "07/01/2026 12:00PM",
targetLastModifiedDate: "07/01/2026 12:00PM",
},
]);
writeEntityFile(tmpDir, targetGuid, "containers", 999, { contentViewID: 999, referenceName: "ref-1" });
// the source pull: one file per live item
for (const id of sourceItemIds) writeEntityFile(tmpDir, sourceGuid, "en-us/item", id, { contentID: id });
return new ContentItemMapper(sourceGuid, targetGuid, "en-us");
}

function linking(to: number): any {
const it = makeContentItem(1);
it.properties.referenceName = "ref-1";
it.properties.definitionName = "TestModel";
it.fields = { title: "A", next: { contentid: to, fulllist: false } };
return it;
}

const logger = () => ({ ...makeLogger(), warning: jest.fn() });

it("sends the item with the link blank, and warns, when the linked item is gone from the source", async () => {
const referenceMapper = seed("gone-src", "gone-tgt", [1]);
const processor = new ContentBatchProcessor(
makeConfig({ sourceGuid: "gone-src", targetGuid: "gone-tgt", referenceMapper })
);
const log = logger();

const result = await (processor as any).prepareContentPayloads([linking(2)], "gone-src", "gone-tgt", log);

expect(result.failedItems).toHaveLength(0);
expect(result.prepared).toHaveLength(1);
expect(JSON.stringify(result.prepared[0].payload.fields.next)).not.toContain("2");
expect(log.warning).toHaveBeenCalledWith(
expect.stringContaining("no longer exist in the source"),
expect.anything()
);
});

it("still fails a link to an item that exists in the source but isn't mapped yet", async () => {
const referenceMapper = seed("live-src", "live-tgt", [1, 2]);
const processor = new ContentBatchProcessor(
makeConfig({ sourceGuid: "live-src", targetGuid: "live-tgt", referenceMapper })
);

const result = await (processor as any).prepareContentPayloads([linking(2)], "live-src", "live-tgt", logger());

expect(result.prepared).toHaveLength(0);
expect(result.failedItems[0].error).toMatch(/Unresolved content reference/);
});

it("doesn't treat a link as gone when no source content was pulled", async () => {
const referenceMapper = seed("nopull-src", "nopull-tgt", []);
const processor = new ContentBatchProcessor(
makeConfig({ sourceGuid: "nopull-src", targetGuid: "nopull-tgt", referenceMapper })
);

const result = await (processor as any).prepareContentPayloads([linking(2)], "nopull-src", "nopull-tgt", logger());

expect(result.prepared).toHaveLength(0);
expect(result.failedItems[0].error).toMatch(/Unresolved content reference/);
});
});

// ─── PROD-2542: payload/source alignment ──────────────────────────────────────
//
// These use the REAL extractContentBatchResults (the module is mocked above for the other tests), so
Expand Down
20 changes: 16 additions & 4 deletions src/lib/verify/canonicalize.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,14 @@
* numbers and numeric strings compare equal, as do booleans and "true"/"false".
*/
import { getLinkedContentCompanionFields } from "../content/linked-content-companion-fields";

/**
* What a context's contentRef returns for a link to an item that no longer exists on its side and
* has no mapping (deleted, e.g. by --syncDeletes along with its copy). Such a link points at nothing,
* so it is dropped, and compares equal to an empty link or to the same kind of link on the other side.
*/
export const DANGLING_CONTENT_REF = "dangling";
const live = (ref: string) => ref !== DANGLING_CONTENT_REF;
import { AssetPrefixPair, stripTrailingSlash } from "./mapping-index";

export type Side = "source" | "target";
Expand Down Expand Up @@ -294,12 +302,14 @@ function canonicalLinkedRef(lowered: Record<string, any>, ctx: CanonicalContext)
let ids: string[] = [];
const sortids = lowered.sortids;
if (typeof sortids === "string" && sortids.trim()) {
ids = parseIdList(sortids).map((id) => ctx.contentRef(id));
ids = parseIdList(sortids)
.map((id) => ctx.contentRef(id))
.filter(live);
} else if (typeof sortids === "number" && sortids > 0) {
ids = [ctx.contentRef(sortids)];
ids = [ctx.contentRef(sortids)].filter(live);
} else {
const contentid = typeof lowered.contentid === "string" ? parseInt(lowered.contentid, 10) : lowered.contentid;
if (typeof contentid === "number" && contentid > 0) ids = [ctx.contentRef(contentid)];
if (typeof contentid === "number" && contentid > 0) ids = [ctx.contentRef(contentid)].filter(live);
}

const fullList = lowered.fulllist === true || String(lowered.fulllist).toLowerCase() === "true";
Expand Down Expand Up @@ -341,7 +351,9 @@ export function canonicalValue(value: any, ctx: CanonicalContext, fieldType?: st
if (typeof value === "string" || typeof value === "number") {
const text = String(value);
if (type === "content" && ID_LIST.test(text)) {
const ids = parseIdList(text).map((id) => ctx.contentRef(id));
const ids = parseIdList(text)
.map((id) => ctx.contentRef(id))
.filter(live);
return ids.length > 0 ? { "@ids": ids } : undefined;
}
if (type === "photogallery" && /^\d+$/.test(text.trim()) && parseInt(text, 10) > 0) {
Expand Down
7 changes: 7 additions & 0 deletions src/lib/verify/tests/check-content.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@ describe("checkContent", () => {
expect(collector.findings).toEqual([]);
});

it("treats a link to an item deleted from the target as equal to an empty link", () => {
// The sync sent 1 with its link to the deleted item 2 blanked; 101 on the target still points at
// its own deleted partner 102. Both point at nothing.
const collector = run([item(1, { title: "A" })], [item(101, { title: "A", related: "102" })], [[1, 101]]);
expect(collector.findings).toEqual([]);
});

it("still fails a link to an unmapped item that exists in the source", () => {
const collector = run(
[item(1, { title: "A", related: { contentid: 2, fulllist: false } }), item(2, { title: "B" })],
Expand Down
9 changes: 1 addition & 8 deletions src/lib/verify/verify-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { GuidDataLoader, GuidEntities, ModelFilterOptions } from "../pushers/gui
import { SitemapHierarchy } from "../pushers/page-pusher/sitemap-hierarchy";
import { SitemapNode } from "../../types/syncAnalysis";
import { LocaleMappingIndex, MappingIndex } from "./mapping-index";
import { AssetUrlTranslator, buildAssetUrlTranslator, CanonicalContext } from "./canonicalize";
import { AssetUrlTranslator, buildAssetUrlTranslator, CanonicalContext, DANGLING_CONTENT_REF } from "./canonicalize";

export const UNPUBLISHED_STATE = 7;

Expand Down Expand Up @@ -220,13 +220,6 @@ export function createVerifyContext(input: {
};
}

/**
* A content reference to an item that no longer exists on its side and has no mapping, e.g. both
* halves of a pair `--syncDeletes` removed. The same token on both sides, so two such references
* compare equal instead of `unmapped:<source id>` against a bare target ID.
*/
const DANGLING_CONTENT_REF = "dangling";

/** Canonicalization context for SOURCE values: every ID is translated to its target. */
export function sourceCanonicalContext(ctx: VerifyContext): CanonicalContext {
return {
Expand Down
Loading