diff --git a/packages/stax-xml/scripts/w3c-canonical.ts b/packages/stax-xml/scripts/w3c-canonical.ts new file mode 100644 index 00000000..9ed19352 --- /dev/null +++ b/packages/stax-xml/scripts/w3c-canonical.ts @@ -0,0 +1,32 @@ +import { XmlEventType, type AnyXmlEvent } from '../src/core/types.js'; + +const escapes: Record = { + '&': '&', '<': '<', '>': '>', '"': '"', + '\t': ' ', '\n': ' ', '\r': ' ', +}; + +function escape(value: string): string { + return value.replace(/[&<>"\t\n\r]/g, char => escapes[char]!); +} + +export function canonicalXml(events: Iterable): string { + let output = ''; + let depth = 0; + for (const event of events) { + if (event.type === XmlEventType.START_ELEMENT) { + const attributes = Array.from(event.attributes?.values() ?? []) + .sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0) + .map(attribute => ` ${attribute.name}="${escape(attribute.value)}"`).join(''); + output += `<${event.name}${attributes}>`; + depth++; + } else if (event.type === XmlEventType.END_ELEMENT) { + output += ``; + depth--; + } else if ((event.type === XmlEventType.CHARACTERS || event.type === XmlEventType.CDATA) && depth > 0) { + output += escape(event.value); + } else if (event.type === XmlEventType.PROCESSING_INSTRUCTION) { + output += ``; + } + } + return output; +} diff --git a/packages/stax-xml/scripts/w3c-xmlts.ts b/packages/stax-xml/scripts/w3c-xmlts.ts index c3f711ca..c0d90301 100644 --- a/packages/stax-xml/scripts/w3c-xmlts.ts +++ b/packages/stax-xml/scripts/w3c-xmlts.ts @@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process'; import { pipeline } from 'node:stream/promises'; import { get } from 'node:https'; import { EventReaderSync } from '../src/sync/index.js'; +import { canonicalXml } from './w3c-canonical.js'; const XMLTS_URL = 'https://www.w3.org/XML/Test/xmlts20130923.zip'; const XMLTS_SHA256 = 'f9510b3532926e1b4c2e54855b021e4b8a66ec98a5337dcf4ff07e8a41968deb'; @@ -51,6 +52,7 @@ interface W3cCase { type: 'valid' | 'invalid' | 'not-wf'; uri: string; path: string; + outputPath?: string; sections: string; namespace: 'yes' | 'no'; } @@ -60,6 +62,8 @@ interface RunResult { failed: Array<{ test: W3cCase; error: string }>; skipped: number; skippedReasons: Map; + outputsChecked: number; + outputsSkipped: number; } await main(); @@ -73,6 +77,8 @@ async function main(): Promise { console.log(`archive: ${XMLTS_URL}`); console.log(`sha256: ${XMLTS_SHA256}`); console.log(`passed: ${result.passed}`); + console.log(`canonical outputs checked: ${result.outputsChecked}`); + console.log(`canonical outputs skipped (DTD processing unsupported): ${result.outputsSkipped}`); console.log(`skipped: ${result.skipped}`); for (const [reason, count] of result.skippedReasons) { console.log(`skipped (${reason}): ${count}`); @@ -187,11 +193,13 @@ function parseManifest(root: string, manifest: string, content: string): W3cCase } const relPath = normalize(join(manifestRelDir, currentBase, uri)); + const output = readAttr(token, 'OUTPUT'); tests.push({ id: readAttr(token, 'ID') ?? relPath, type, uri: relPath, path: join(root, relPath), + outputPath: output ? join(root, manifestRelDir, currentBase, output) : undefined, sections: readAttr(token, 'SECTIONS') ?? '', namespace: readAttr(token, 'NAMESPACE') === 'no' ? 'no' : 'yes' }); @@ -238,7 +246,7 @@ function isInGateScope(test: W3cCase): boolean { } function runTests(tests: W3cCase[]): RunResult { - const result: RunResult = { passed: 0, failed: [], skipped: 0, skippedReasons: new Map() }; + const result: RunResult = { passed: 0, failed: [], skipped: 0, skippedReasons: new Map(), outputsChecked: 0, outputsSkipped: 0 }; for (const test of tests) { let bytes: Buffer; @@ -290,15 +298,36 @@ function runTests(tests: W3cCase[]): RunResult { } let threw = false; + let output = ''; let error = ''; try { - Array.from(new EventReaderSync(bytes, { documentMode: 'document', encoding })); + output = canonicalXml(new EventReaderSync(bytes, { documentMode: 'document', encoding })); } catch (cause) { threw = true; error = cause instanceof Error ? cause.message : String(cause); } const shouldThrow = test.type === 'not-wf'; + if (!threw && !shouldThrow && test.outputPath) { + try { + const expected = readFileSync(test.outputPath, 'utf8'); + if (/]+\s+(?:SYSTEM|PUBLIC)\b/i.test(xml) || expected.includes('= this.currentAttributeCount) return undefined; - const value = this.buffer.slice( + const value = normalizeAttributeWhitespace(this.buffer.slice( this.attributeValueStarts[index]!, this.attributeValueEnds[index]!, - ); + )); return this.autoDecodeEntities ? decodeEntities(value, this.customEntities, true) : value; @@ -365,9 +366,11 @@ export class TokenCursor { } this.xmlDeclarationAllowed = false; this.currentNameValue = target; - this.currentTextMemo = this.buffer.slice(targetEnd, end).trim(); - this.currentTextStart = this.currentTextEnd = 0; - return this.set(XmlEventType.PROCESSING_INSTRUCTION, true); + while (targetEnd < end && isXmlWhitespace(this.buffer.charCodeAt(targetEnd))) + targetEnd++; + this.currentTextStart = targetEnd; + this.currentTextEnd = end; + return this.set(XmlEventType.PROCESSING_INSTRUCTION); } private parseBang(): TokenCursorResult { @@ -661,10 +664,10 @@ export class TokenCursor { const declaredPrefix = colon < 0 ? "" : this.buffer.slice(colon + 1, end); const value = decodeEntities( - this.buffer.slice( + normalizeAttributeWhitespace(this.buffer.slice( this.attributeValueStarts[index]!, this.attributeValueEnds[index]!, - ), + )), this.customEntities, true, ); @@ -871,19 +874,7 @@ export class TokenCursor { const name = this.buffer.slice(start, end); const attribute = { name, - value: this.autoDecodeEntities - ? decodeEntities( - this.buffer.slice( - this.attributeValueStarts[index]!, - this.attributeValueEnds[index]!, - ), - this.customEntities, - true, - ) - : this.buffer.slice( - this.attributeValueStarts[index]!, - this.attributeValueEnds[index]!, - ), + value: this.attributeValue(index)!, localName: colon < 0 ? name : this.buffer.slice(colon + 1, end), prefix: colon < 0 ? "" : this.buffer.slice(start, colon), namespaceURI: this.attributeNamespaceURIs[index]!, @@ -1280,10 +1271,7 @@ export class TokenCursor { if (this.final) throw new Error(`Unterminated ${kind}.`); return NEED_INPUT; } - private set( - type: XmlEventTypeValue, - preserveTextMemo = false, - ): XmlEventTypeValue { + private set(type: XmlEventTypeValue): XmlEventTypeValue { for (let index = 0; index < this.attributeMemoHighWater; index++) this.attributeMemos[index] = undefined; this.attributeMemoHighWater = 0; @@ -1300,7 +1288,7 @@ export class TokenCursor { ) { this.currentTextStart = this.currentTextEnd = NO_SPAN; this.currentTextMemo = undefined; - } else if (!preserveTextMemo) { + } else { this.currentTextMemo = undefined; } if ( @@ -1353,6 +1341,10 @@ function compileCustomEntities( return entities; } +function normalizeAttributeWhitespace(value: string): string { + return /[\t\r\n]/.test(value) ? value.replace(/\r\n|[\t\r\n]/g, " ") : value; +} + function decodeEntities( value: string, customEntities?: ReadonlyMap, diff --git a/packages/stax-xml/test/converter/parser-edge-cases.test.ts b/packages/stax-xml/test/converter/parser-edge-cases.test.ts index 4f9fdf77..6f1d0abc 100644 --- a/packages/stax-xml/test/converter/parser-edge-cases.test.ts +++ b/packages/stax-xml/test/converter/parser-edge-cases.test.ts @@ -2,6 +2,16 @@ import { describe, expect, it } from 'vitest'; import { x } from '../../src/converter/converter/index.js'; describe('Parser Edge Cases Coverage', () => { + it('normalizes literal whitespace while preserving character references', () => { + const schema = x.object({ + text: x.string().xpath('/a/t'), + attr: x.string().xpath('/a/@b'), + }); + expect(schema.parseSync('x\r\ny')).toEqual({ + text: 'x\ny', + attr: '1 2 3\r', + }); + }); describe('Deep transform nesting', () => { it('should handle 5 levels of transforms', () => { const xml = 'test'; diff --git a/packages/stax-xml/test/token-cursor-resume.test.ts b/packages/stax-xml/test/token-cursor-resume.test.ts index 75e55c19..e0145007 100644 --- a/packages/stax-xml/test/token-cursor-resume.test.ts +++ b/packages/stax-xml/test/token-cursor-resume.test.ts @@ -45,6 +45,30 @@ function collect(xml: string, chunkSize?: number): unknown[][] { } describe('TokenCursor resumable token scans', () => { + it('normalizes literal line endings and attribute whitespace before decoding references', () => { + const xml = 'x\r\ny\rz '; + const expected = [ + [XmlEventType.START_ELEMENT, 'r', [['a', '1 2 3 4 5\t\n\r']]], + [XmlEventType.CHARACTERS, 'x\ny\nz\r'], + [XmlEventType.CDATA, 'c\nd\ne'], + [XmlEventType.COMMENT, 'f\ng'], + ]; + for (const chunkSize of [undefined, 1, 2, 7]) { + const tokens = collect(xml, chunkSize); + for (const token of expected) expect(tokens).toContainEqual(token); + } + }); + + it('normalizes namespace values and leaves references encoded when decoding is disabled', () => { + const cursor = new TokenCursor('x\r\ny ', true, { autoDecodeEntities: false }); + expect(cursor.next()).toBe(XmlEventType.START_DOCUMENT); + expect(cursor.next()).toBe(XmlEventType.START_ELEMENT); + expect(cursor.attributeValue(1)).toBe('x '); + expect(cursor.next()).toBe(XmlEventType.CHARACTERS); + expect(cursor.text()).toBe('x\ny '); + expect(cursor.next()).toBe(XmlEventType.START_ELEMENT); + expect(cursor.namespaceURI()).toBe('urn:one two'); + }); it('matches complete input when long tokens cross one-code-unit chunks', () => { const payload = 'x'.repeat(16_384); const dtdPadding = ' '.repeat(16_384); diff --git a/packages/stax-xml/test/v1-reader-contract.test.ts b/packages/stax-xml/test/v1-reader-contract.test.ts index d635696e..0672843f 100644 --- a/packages/stax-xml/test/v1-reader-contract.test.ts +++ b/packages/stax-xml/test/v1-reader-contract.test.ts @@ -180,6 +180,34 @@ describe("v1 reader public surface", () => { }); describe("v1 current-token reader contract", () => { + it("preserves trailing processing instruction data and normalizes line endings", async () => { + const xml = ''; + const expected = [XmlEventType.PROCESSING_INSTRUCTION, "work", "first\nsecond\nthird \t"]; + expect(collectSync(xml)).toContainEqual(expected); + expect(collectSync(chunks(xml))).toContainEqual(expected); + expect(await collectAsync(asyncChunks(xml))).toContainEqual(expected); + expect(Array.from(new EventReaderSync(xml)).map(eventToken)).toContainEqual(expected); + const events: unknown[][] = []; + for await (const event of new EventReader(asyncChunks(xml))) events.push(eventToken(event)); + expect(events).toContainEqual(expected); + }); + it("normalizes line endings and literal attribute whitespace across reader inputs", async () => { + const xml = 'x\r\ny'; + const expected = collectSync(xml); + expect(expected).toContainEqual([XmlEventType.CHARACTERS, "x\ny"]); + expect(expected).toContainEqual([XmlEventType.CDATA, "z\nw"]); + expect(expected).toContainEqual([ + XmlEventType.START_ELEMENT, "r", "r", "", "", + [["a", "a", "", "", "1 2 3\r"]], + ]); + expect(collectSync(encoder.encode(xml))).toEqual(expected); + expect(collectSync(chunks(xml))).toEqual(expected); + await expect(collectAsync(asyncChunks(xml))).resolves.toEqual(expected); + expect(Array.from(new EventReaderSync(xml)).map(eventToken)).toEqual(expected); + const events: unknown[][] = []; + for await (const event of new EventReader(asyncChunks(xml))) events.push(eventToken(event)); + expect(events).toEqual(expected); + }); it("uses the configured fatal TextDecoder encoding for byte readers", async () => { const xml = '한글'; const expected = collectSync(xml); diff --git a/packages/stax-xml/test/w3c-canonical.test.ts b/packages/stax-xml/test/w3c-canonical.test.ts new file mode 100644 index 00000000..82cd84de --- /dev/null +++ b/packages/stax-xml/test/w3c-canonical.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from 'vitest'; +import { canonicalXml } from '../scripts/w3c-canonical.js'; +import { EventReaderSync } from '../src/sync/index.js'; + +describe('W3C canonical output', () => { + it('serializes normalized values, sorted attributes, CDATA, and processing instructions', () => { + expect(canonicalXml(new EventReaderSync('a\r\nb]]>'))) + .toBe('a b<c>'); + expect(canonicalXml(new EventReaderSync('x\r\ny'))).not.toBe('x y'); + }); +}); diff --git a/packages/stax-xml/vitest.unit.config.ts b/packages/stax-xml/vitest.unit.config.ts index 66541517..6a4a9518 100644 --- a/packages/stax-xml/vitest.unit.config.ts +++ b/packages/stax-xml/vitest.unit.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ }, test: { include: [ + "test/w3c-canonical.test.ts", "test/v1-reader-contract.test.ts", "test/token-cursor-resume.test.ts", "test/token-cursor-options-coverage.test.ts",