diff --git a/deno.json b/deno.json index a525351..a9cf7f3 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@aralroca/diff-dom-streaming", - "version": "0.6.6", + "version": "0.6.8", "exports": { ".": "./src/index.ts", "./types": "./index.d.ts" diff --git a/package.json b/package.json index 23604ae..299b9ed 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "diff-dom-streaming", - "version": "0.6.7", + "version": "0.6.8", "bugs": "https://github.com/aralroca/diff-dom-streaming/issues", "description": "Diff DOM algorithm with streaming. Gets all modifications, insertions and removals between a DOM fragment and a stream HTML reader.", "keywords": [ diff --git a/src/index.test.ts b/src/index.test.ts index 2b3e294..e9b25f4 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1765,11 +1765,282 @@ describe("Diff test", () => { `), ); }); + + it("should diff correctly when tag and attribute names are split across chunks", async () => { + const [newHTML] = await testDiff({ + oldHTMLString: ` +
adios
+ `, + newHTMLStringChunks: ["hola"], + slowChunks: true, + }); + + expect(newHTML).toBe( + normalize(` + + + +
hola
+ + + `), + ); + }); + + it("should diff correctly when an attribute value is split across chunks", async () => { + const [newHTML] = await testDiff({ + oldHTMLString: ` +
x
+ `, + newHTMLStringChunks: ['
x
'], + slowChunks: true, + }); + + expect(newHTML).toBe( + normalize(` + + + +
x
+ + + `), + ); + }); + + it("should decode multi-byte UTF-8 characters split across chunk boundaries", async () => { + const encoded = Array.from( + new TextEncoder().encode("
mañana 🚀
"), + ); + + // "ñ" is 2 bytes starting at index 7 → cutting at 8 splits it in half. + // "🚀" is 4 bytes starting at index 13 → cutting at 16 splits it apart. + expect(encoded.slice(7, 9)).toEqual([0xc3, 0xb1]); + expect(encoded.slice(13, 17)).toEqual([0xf0, 0x9f, 0x9a, 0x80]); + + const [newHTML] = await testDiff({ + oldHTMLString: ` +
old
+ `, + newHTMLByteChunks: [ + encoded.slice(0, 8), + encoded.slice(8, 16), + encoded.slice(16), + ], + slowChunks: true, + }); + + expect(newHTML).toBe( + normalize(` + + + +
mañana 🚀
+ + + `), + ); + }); + + // Pins the CURRENT behavior: new ", + ], + }); + const executed = await page.evaluate( + () => (window as any).__executed === true, + ); + + expect(newHTML).toBe( + normalize(` + + + +
foo
+ + + + `), + ); + expect(executed).toBeFalse(); + }); + + it("should NOT re-execute a pre-existing
foo
`, + newHTMLStringChunks: [ + "", + "
bar
", + ], + }); + const count = await page.evaluate(() => (window as any).__count); + + expect(newHTML).toBe( + normalize(` + + + + + +
bar
+ + + `), + ); + // Executed exactly once (on initial page load), not again after diffing. + expect(count).toBe(1); + }); + + it("should reject the diff when the stream errors after a chunk instead of hanging the walker", async () => { + await expect( + testDiff({ + oldHTMLString: ` +
foo
+ `, + newHTMLStringChunks: ["
bar
"], + errorStreamMessage: "network dead", + slowChunks: true, + }), + ).rejects.toThrow("network dead"); + }); + + it("should reject the diff when the stream errors before any chunk", async () => { + await expect( + testDiff({ + oldHTMLString: ` +
foo
+ `, + newHTMLStringChunks: [], + errorStreamMessage: "network dead", + }), + ).rejects.toThrow("network dead"); + }); + + it("should apply children of an open last node progressively (main as last child)", async () => { + const [newHTML, , , , midStreamH1] = await testDiff({ + oldHTMLString: ` +
+ +

OLD

para OLD

tail

+
+ `, + newHTMLStringChunks: [ + '

NEW

para NEW

', + "

tail

", + ], + midStreamEval: `document.querySelector('h1').textContent`, + }); + + // The
subtree must be diffed WHILE the stream is still open, + // even though
is the last child of a still-open container. + expect(midStreamH1).toBe("NEW"); + expect(newHTML).toBe( + normalize(` + + + +
+ +

NEW

para NEW

tail

+
+ + + `), + ); + }); + + it("should apply an open last node progressively and still ADD trailing siblings from a later chunk", async () => { + const [newHTML, , , , midStreamH1] = await testDiff({ + oldHTMLString: ` +
+ +

OLD

tail

+
+ `, + newHTMLStringChunks: [ + '

NEW

tail

', + "
bye
", + ], + midStreamEval: `document.querySelector('h1').textContent`, + }); + + expect(midStreamH1).toBe("NEW"); + expect(newHTML).toBe( + normalize(` + + + +
+ +

NEW

tail

+
bye
+
+ + + `), + ); + }); + + it("should defer pruning of trailing old nodes until the open level is closed", async () => { + const [newHTML, , , , midStream] = await testDiff({ + oldHTMLString: ` +
+ +

OLD

tail

+
bye
+ +
+ `, + newHTMLStringChunks: [ + '

NEW

tail

', + "
", + ], + midStreamEval: `JSON.stringify({ + h1: document.querySelector('h1').textContent, + footer: !!document.querySelector('footer'), + aside: !!document.querySelector('aside'), + })`, + }); + + // Mid-stream: the open subtree is already applied, but the trailing old + // nodes are NOT pruned yet — the level may still receive new children. + expect(JSON.parse(midStream)).toEqual({ + h1: "NEW", + footer: true, + aside: true, + }); + // Once the stream closes, the deferred pruning runs. + expect(newHTML).toBe( + normalize(` + + + +
+ +

NEW

tail

+
+ + + `), + ); + }); }); async function testDiff({ oldHTMLString, - newHTMLStringChunks, + newHTMLStringChunks = [], + newHTMLByteChunks, + errorStreamMessage, + midStreamEval, useForEeachStreamNode = false, slowChunks = false, transition = false, @@ -1778,19 +2049,32 @@ describe("Diff test", () => { onNextNode, }: { oldHTMLString: string; - newHTMLStringChunks: string[]; + newHTMLStringChunks?: string[]; + // Raw byte chunks (each an array of UTF-8 byte values) to test chunk + // boundaries that split multi-byte characters, which cannot be expressed + // as JS strings. + newHTMLByteChunks?: number[][]; + // When set, the stream errors with this message after emitting all chunks + // instead of closing (simulates an aborted fetch / dead network). + errorStreamMessage?: string; + // JS expression evaluated in the page BEFORE enqueuing the last chunk + // (after a settle delay), to observe the mid-stream DOM state. + midStreamEval?: string; useForEeachStreamNode?: boolean; slowChunks?: boolean; transition?: boolean; ignoreId?: boolean; registerWC?: boolean; onNextNode?: string - }): Promise<[string, any[], Node[], boolean, string]> { + }): Promise<[string, any[], Node[], boolean, any, string]> { await page.setContent(normalize(oldHTMLString)); - const [mutations, streamNodes, transitionApplied, logs] = await page.evaluate( + const [mutations, streamNodes, transitionApplied, midStreamResult, logs] = await page.evaluate( async ([ diffCode, newHTMLStringChunks, + newHTMLByteChunks, + errorStreamMessage, + midStreamEval, useForEeachStreamNode, slowChunks, transition, @@ -1800,14 +2084,36 @@ describe("Diff test", () => { ]) => { eval(diffCode as string); const encoder = new TextEncoder(); + const byteChunks = newHTMLByteChunks as number[][] | undefined; + const chunks = byteChunks ?? (newHTMLStringChunks as string[]); + let midStreamResult; const readable = new ReadableStream({ start: async (controller) => { - for (const chunk of newHTMLStringChunks as string[]) { + for (let i = 0; i < chunks.length; i++) { + if (slowChunks) + await new Promise((resolve) => setTimeout(resolve, 100)); + if (midStreamEval && i === chunks.length - 1) { + // Let the walker settle on the already-emitted chunks before + // observing the mid-stream DOM state. + await new Promise((resolve) => setTimeout(resolve, 500)); + midStreamResult = eval(midStreamEval as string); + } + const chunk = chunks[i]; + controller.enqueue( + byteChunks + ? new Uint8Array(chunk as number[]) + : encoder.encode(chunk as string), + ); + } + if (errorStreamMessage) { + // Let the already-enqueued chunks be consumed by the walker + // first, so the error lands while it awaits the next chunk. if (slowChunks) await new Promise((resolve) => setTimeout(resolve, 100)); - controller.enqueue(encoder.encode(chunk)); + controller.error(new Error(errorStreamMessage as string)); + } else { + controller.close(); } - controller.close(); }, }); const allMutations: any[] = []; @@ -1881,11 +2187,14 @@ describe("Diff test", () => { observer.disconnect(); - return [allMutations, streamNodes, transitionApplied, (window as any).logs]; + return [allMutations, streamNodes, transitionApplied, midStreamResult, (window as any).logs]; }, [ diffCode, newHTMLStringChunks, + newHTMLByteChunks, + errorStreamMessage, + midStreamEval, useForEeachStreamNode, slowChunks, transition, @@ -1900,6 +2209,7 @@ describe("Diff test", () => { mutations, streamNodes, transitionApplied, + midStreamResult, logs ]; } diff --git a/src/index.ts b/src/index.ts index e8c28da..ebae025 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ type Walker = { [NEXT_SIBLING]: (node: Node) => Promise; [APPLY_TRANSITION]: (v: () => void) => void; [VISITED]: WeakSet; + [SETTLE]: (node: Node) => Promise; }; type NextNodeCallback = (node: Node) => void; @@ -30,6 +31,7 @@ const APPLY_TRANSITION = 0; const FIRST_CHILD = 1; const NEXT_SIBLING = 2; const VISITED = 3; +const SETTLE = 4; const SPECIAL_TAGS = new Set(["HTML", "HEAD", "BODY"]); const wait = () => new Promise((resolve) => requestAnimationFrame(resolve)); @@ -101,6 +103,7 @@ function settledWalker(walker: Walker, options: Options = {}): Walker { [NEXT_SIBLING]: hop("nextSibling"), [APPLY_TRANSITION]: (v) => v(), [VISITED]: walker[VISITED], + [SETTLE]: async () => {}, }; } @@ -109,6 +112,7 @@ function settledWalker(walker: Walker, options: Options = {}): Walker { */ async function updateNode(oldNode: Node, newNode: Node, walker: Walker) { if (oldNode.nodeType !== newNode.nodeType) { + await walker[SETTLE](newNode); markSubtree(newNode, walker[VISITED]); return walker[APPLY_TRANSITION](() => @@ -232,6 +236,7 @@ async function setChildNodes(oldParent: Node, newParent: Node, walker: Walker) { checkOld = oldNode; oldNode = oldNode.nextSibling; if (getKey(checkOld)) { + await walker[SETTLE](newNode); markSubtree(newNode, walker[VISITED]); insertedNode = newNode.cloneNode(true); walker[APPLY_TRANSITION](() => @@ -241,6 +246,7 @@ async function setChildNodes(oldParent: Node, newParent: Node, walker: Walker) { await updateNode(checkOld, newNode, walker); } } else { + await walker[SETTLE](newNode); markSubtree(newNode, walker[VISITED]); insertedNode = newNode.cloneNode(true); walker[APPLY_TRANSITION](() => oldParent.appendChild(insertedNode!)); @@ -282,42 +288,65 @@ async function htmlStreamWalker( const decoderStream = new TextDecoderStream(); const decoderStreamReader = decoderStream.readable.getReader(); let streamInProgress = true; + let streamError: Error | undefined; - stream.pipeTo(decoderStream.writable); + // The error already surfaces through the reader in processStream; this + // only silences the duplicated unhandled rejection. + stream.pipeTo(decoderStream.writable).catch(() => {}); processStream(); async function processStream() { try { while (true) { const { done, value } = await decoderStreamReader.read(); - if (done) { - streamInProgress = false; - break; - } + if (done) break; doc.write(value); } + } catch (error) { + streamError = error as Error; } finally { + streamInProgress = false; doc.close(); } } - while (!doc.documentElement || isLastNodeOfChunk(doc.documentElement)) { + while ( + !streamError && + (!doc.documentElement || isLastNodeOfChunk(doc.documentElement)) + ) { await wait(); } + if (streamError) throw streamError; + const visited = new WeakSet(); function next(field: "firstChild" | "nextSibling") { return async (node: Node) => { if (!node) return null; - let nextNode = node[field]; + const hop = () => { + let nextNode = node[field]; + + // Hop over ignored nodes by SIBLING: hopping by `field` would descend + // INTO an ignored first child instead of skipping it. + while (options.shouldIgnoreNode?.(nextNode)) { + nextNode = nextNode!.nextSibling; + } - // Hop over ignored nodes by SIBLING: hopping by `field` would descend - // INTO an ignored first child instead of skipping it. - while (options.shouldIgnoreNode?.(nextNode)) { - nextNode = nextNode!.nextSibling; + return nextNode; + }; + + let nextNode = hop(); + + // A null hop FROM the parser's frontier is not the end of the level: + // the parent is still open, so later chunks may still add nodes here. + // Waiting defers the pruning at the end of setChildNodes until the + // level is closed (or the stream ends/errors). + while (!nextNode && isLastNodeOfChunk(node)) { + await wait(); + nextNode = hop(); } if (nextNode) { @@ -325,12 +354,16 @@ async function htmlStreamWalker( await options.onNextNode?.(nextNode); } - const waitChildren = field === "firstChild"; - - while (isLastNodeOfChunk(nextNode as Element, waitChildren)) { + // Wait only while the node is the parser's frontier AND has no children + // yet: a frontier node with children can already be diffed progressively + // (its own hops wait as needed), instead of stalling the whole subtree + // until the stream closes. + while (isLastNodeOfChunk(nextNode as Element, true)) { await wait(); } + if (streamError) throw streamError; + return nextNode; }; } @@ -372,5 +405,14 @@ async function htmlStreamWalker( } else v(); }, [VISITED]: visited, + // Waits until the node stops being the parser's frontier (or the stream + // ends), so cloning it deeply cannot snapshot a half-parsed subtree. + [SETTLE]: async (node: Node) => { + while (isLastNodeOfChunk(node)) { + await wait(); + } + + if (streamError) throw streamError; + }, }; }