Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions packages/stax-xml/scripts/w3c-canonical.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { XmlEventType, type AnyXmlEvent } from '../src/core/types.js';

const escapes: Record<string, string> = {
'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;',
'\t': '&#9;', '\n': '&#10;', '\r': '&#13;',
};

function escape(value: string): string {
return value.replace(/[&<>"\t\n\r]/g, char => escapes[char]!);
}

export function canonicalXml(events: Iterable<AnyXmlEvent>): string {
let output = '';
let depth = 0;
for (const event of events) {
if (event.type === XmlEventType.START_ELEMENT) {
const attributes = Array.from(event.attributes?.values() ?? [])
.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)
.map(attribute => ` ${attribute.name}="${escape(attribute.value)}"`).join('');
output += `<${event.name}${attributes}>`;
depth++;
} else if (event.type === XmlEventType.END_ELEMENT) {
output += `</${event.name}>`;
depth--;
} else if ((event.type === XmlEventType.CHARACTERS || event.type === XmlEventType.CDATA) && depth > 0) {
output += escape(event.value);
} else if (event.type === XmlEventType.PROCESSING_INSTRUCTION) {
output += `<?${event.target} ${event.data}?>`;
}
}
return output;
}
33 changes: 31 additions & 2 deletions packages/stax-xml/scripts/w3c-xmlts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { spawnSync } from 'node:child_process';
import { pipeline } from 'node:stream/promises';
import { get } from 'node:https';
import { EventReaderSync } from '../src/sync/index.js';
import { canonicalXml } from './w3c-canonical.js';

const XMLTS_URL = 'https://www.w3.org/XML/Test/xmlts20130923.zip';
const XMLTS_SHA256 = 'f9510b3532926e1b4c2e54855b021e4b8a66ec98a5337dcf4ff07e8a41968deb';
Expand Down Expand Up @@ -51,6 +52,7 @@ interface W3cCase {
type: 'valid' | 'invalid' | 'not-wf';
uri: string;
path: string;
outputPath?: string;
sections: string;
namespace: 'yes' | 'no';
}
Expand All @@ -60,6 +62,8 @@ interface RunResult {
failed: Array<{ test: W3cCase; error: string }>;
skipped: number;
skippedReasons: Map<string, number>;
outputsChecked: number;
outputsSkipped: number;
}

await main();
Expand All @@ -73,6 +77,8 @@ async function main(): Promise<void> {
console.log(`archive: ${XMLTS_URL}`);
console.log(`sha256: ${XMLTS_SHA256}`);
console.log(`passed: ${result.passed}`);
console.log(`canonical outputs checked: ${result.outputsChecked}`);
console.log(`canonical outputs skipped (DTD processing unsupported): ${result.outputsSkipped}`);
console.log(`skipped: ${result.skipped}`);
for (const [reason, count] of result.skippedReasons) {
console.log(`skipped (${reason}): ${count}`);
Expand Down Expand Up @@ -187,11 +193,13 @@ function parseManifest(root: string, manifest: string, content: string): W3cCase
}

const relPath = normalize(join(manifestRelDir, currentBase, uri));
const output = readAttr(token, 'OUTPUT');
tests.push({
id: readAttr(token, 'ID') ?? relPath,
type,
uri: relPath,
path: join(root, relPath),
outputPath: output ? join(root, manifestRelDir, currentBase, output) : undefined,
sections: readAttr(token, 'SECTIONS') ?? '',
namespace: readAttr(token, 'NAMESPACE') === 'no' ? 'no' : 'yes'
});
Expand Down Expand Up @@ -238,7 +246,7 @@ function isInGateScope(test: W3cCase): boolean {
}

function runTests(tests: W3cCase[]): RunResult {
const result: RunResult = { passed: 0, failed: [], skipped: 0, skippedReasons: new Map() };
const result: RunResult = { passed: 0, failed: [], skipped: 0, skippedReasons: new Map(), outputsChecked: 0, outputsSkipped: 0 };

for (const test of tests) {
let bytes: Buffer;
Expand Down Expand Up @@ -290,15 +298,36 @@ function runTests(tests: W3cCase[]): RunResult {
}

let threw = false;
let output = '';
let error = '';
try {
Array.from(new EventReaderSync(bytes, { documentMode: 'document', encoding }));
output = canonicalXml(new EventReaderSync(bytes, { documentMode: 'document', encoding }));
} catch (cause) {
threw = true;
error = cause instanceof Error ? cause.message : String(cause);
}

const shouldThrow = test.type === 'not-wf';
if (!threw && !shouldThrow && test.outputPath) {
try {
const expected = readFileSync(test.outputPath, 'utf8');
if (/<!ATTLIST\b|<!ENTITY\s+%|<!DOCTYPE\s+[^\s\[>]+\s+(?:SYSTEM|PUBLIC)\b/i.test(xml) || expected.includes('<!DOCTYPE')) {
result.outputsSkipped++;
} else {
result.outputsChecked++;
if (output !== expected) {
let index = 0;
while (index < output.length && output[index] === expected[index]) index++;
const start = Math.max(0, index - 30);
result.failed.push({ test, error: `canonical output differs near offset ${start}: expected ${JSON.stringify(expected.slice(start, start + 120))}, got ${JSON.stringify(output.slice(start, start + 120))}` });
continue;
}
}
} catch (cause) {
result.failed.push({ test, error: `cannot read canonical output: ${String(cause)}` });
continue;
}
}
if (threw === shouldThrow) {
result.passed++;
} else {
Expand Down
44 changes: 18 additions & 26 deletions packages/stax-xml/src/core/TokenCursor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,10 +212,11 @@ export class TokenCursor {
text(): string | undefined {
if (this.currentTextStart < 0) return undefined;
if (this.currentTextMemo === undefined) {
const value = this.buffer.slice(
let value = this.buffer.slice(
this.currentTextStart,
this.currentTextEnd,
);
if (value.includes("\r")) value = value.replace(/\r\n?/g, "\n");
this.currentTextMemo =
this.currentType === XmlEventType.CHARACTERS && this.autoDecodeEntities
? decodeEntities(value, this.customEntities)
Expand Down Expand Up @@ -250,10 +251,10 @@ export class TokenCursor {
}
attributeValue(index: number): string | undefined {
if (index < 0 || index >= this.currentAttributeCount) return undefined;
const value = this.buffer.slice(
const value = normalizeAttributeWhitespace(this.buffer.slice(
this.attributeValueStarts[index]!,
this.attributeValueEnds[index]!,
);
));
return this.autoDecodeEntities
? decodeEntities(value, this.customEntities, true)
: value;
Expand Down Expand Up @@ -365,9 +366,11 @@ export class TokenCursor {
}
this.xmlDeclarationAllowed = false;
this.currentNameValue = target;
this.currentTextMemo = this.buffer.slice(targetEnd, end).trim();
this.currentTextStart = this.currentTextEnd = 0;
return this.set(XmlEventType.PROCESSING_INSTRUCTION, true);
while (targetEnd < end && isXmlWhitespace(this.buffer.charCodeAt(targetEnd)))
targetEnd++;
this.currentTextStart = targetEnd;
this.currentTextEnd = end;
return this.set(XmlEventType.PROCESSING_INSTRUCTION);
}

private parseBang(): TokenCursorResult {
Expand Down Expand Up @@ -661,10 +664,10 @@ export class TokenCursor {
const declaredPrefix =
colon < 0 ? "" : this.buffer.slice(colon + 1, end);
const value = decodeEntities(
this.buffer.slice(
normalizeAttributeWhitespace(this.buffer.slice(
this.attributeValueStarts[index]!,
this.attributeValueEnds[index]!,
),
)),
this.customEntities,
true,
);
Expand Down Expand Up @@ -871,19 +874,7 @@ export class TokenCursor {
const name = this.buffer.slice(start, end);
const attribute = {
name,
value: this.autoDecodeEntities
? decodeEntities(
this.buffer.slice(
this.attributeValueStarts[index]!,
this.attributeValueEnds[index]!,
),
this.customEntities,
true,
)
: this.buffer.slice(
this.attributeValueStarts[index]!,
this.attributeValueEnds[index]!,
),
value: this.attributeValue(index)!,
localName: colon < 0 ? name : this.buffer.slice(colon + 1, end),
prefix: colon < 0 ? "" : this.buffer.slice(start, colon),
namespaceURI: this.attributeNamespaceURIs[index]!,
Expand Down Expand Up @@ -1280,10 +1271,7 @@ export class TokenCursor {
if (this.final) throw new Error(`Unterminated ${kind}.`);
return NEED_INPUT;
}
private set(
type: XmlEventTypeValue,
preserveTextMemo = false,
): XmlEventTypeValue {
private set(type: XmlEventTypeValue): XmlEventTypeValue {
for (let index = 0; index < this.attributeMemoHighWater; index++)
this.attributeMemos[index] = undefined;
this.attributeMemoHighWater = 0;
Expand All @@ -1300,7 +1288,7 @@ export class TokenCursor {
) {
this.currentTextStart = this.currentTextEnd = NO_SPAN;
this.currentTextMemo = undefined;
} else if (!preserveTextMemo) {
} else {
this.currentTextMemo = undefined;
}
if (
Expand Down Expand Up @@ -1353,6 +1341,10 @@ function compileCustomEntities(
return entities;
}

function normalizeAttributeWhitespace(value: string): string {
return /[\t\r\n]/.test(value) ? value.replace(/\r\n|[\t\r\n]/g, " ") : value;
}

function decodeEntities(
value: string,
customEntities?: ReadonlyMap<string, string>,
Expand Down
10 changes: 10 additions & 0 deletions packages/stax-xml/test/converter/parser-edge-cases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@ import { describe, expect, it } from 'vitest';
import { x } from '../../src/converter/converter/index.js';

describe('Parser Edge Cases Coverage', () => {
it('normalizes literal whitespace while preserving character references', () => {
const schema = x.object({
text: x.string().xpath('/a/t'),
attr: x.string().xpath('/a/@b'),
});
expect(schema.parseSync('<a b="1\t2\r\n3&#13;"><t>x\r\ny</t></a>')).toEqual({
text: 'x\ny',
attr: '1 2 3\r',
});
});
describe('Deep transform nesting', () => {
it('should handle 5 levels of transforms', () => {
const xml = '<root><value>test</value></root>';
Expand Down
24 changes: 24 additions & 0 deletions packages/stax-xml/test/token-cursor-resume.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,30 @@ function collect(xml: string, chunkSize?: number): unknown[][] {
}

describe('TokenCursor resumable token scans', () => {
it('normalizes literal line endings and attribute whitespace before decoding references', () => {
const xml = '<r a="1\t2\n3\r\n4\r5&#9;&#10;&#13;">x\r\ny\rz&#13;<![CDATA[c\r\nd\re]]><!--f\r\ng--></r>';
const expected = [
[XmlEventType.START_ELEMENT, 'r', [['a', '1 2 3 4 5\t\n\r']]],
[XmlEventType.CHARACTERS, 'x\ny\nz\r'],
[XmlEventType.CDATA, 'c\nd\ne'],
[XmlEventType.COMMENT, 'f\ng'],
];
for (const chunkSize of [undefined, 1, 2, 7]) {
const tokens = collect(xml, chunkSize);
for (const token of expected) expect(tokens).toContainEqual(token);
}
});

it('normalizes namespace values and leaves references encoded when decoding is disabled', () => {
const cursor = new TokenCursor('<r xmlns:p="urn:one\r\ntwo" a="x\t&#9;">x\r\ny&#13;<p:c/></r>', true, { autoDecodeEntities: false });
expect(cursor.next()).toBe(XmlEventType.START_DOCUMENT);
expect(cursor.next()).toBe(XmlEventType.START_ELEMENT);
expect(cursor.attributeValue(1)).toBe('x &#9;');
expect(cursor.next()).toBe(XmlEventType.CHARACTERS);
expect(cursor.text()).toBe('x\ny&#13;');
expect(cursor.next()).toBe(XmlEventType.START_ELEMENT);
expect(cursor.namespaceURI()).toBe('urn:one two');
});
it('matches complete input when long tokens cross one-code-unit chunks', () => {
const payload = 'x'.repeat(16_384);
const dtdPadding = ' '.repeat(16_384);
Expand Down
28 changes: 28 additions & 0 deletions packages/stax-xml/test/v1-reader-contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,34 @@ describe("v1 reader public surface", () => {
});

describe("v1 current-token reader contract", () => {
it("preserves trailing processing instruction data and normalizes line endings", async () => {
const xml = '<?work first\r\nsecond\rthird \t?><root/>';
const expected = [XmlEventType.PROCESSING_INSTRUCTION, "work", "first\nsecond\nthird \t"];
expect(collectSync(xml)).toContainEqual(expected);
expect(collectSync(chunks(xml))).toContainEqual(expected);
expect(await collectAsync(asyncChunks(xml))).toContainEqual(expected);
expect(Array.from(new EventReaderSync(xml)).map(eventToken)).toContainEqual(expected);
const events: unknown[][] = [];
for await (const event of new EventReader(asyncChunks(xml))) events.push(eventToken(event));
expect(events).toContainEqual(expected);
});
it("normalizes line endings and literal attribute whitespace across reader inputs", async () => {
const xml = '<r a="1\t2\r\n3&#13;">x\r\ny<![CDATA[z\r\nw]]></r>';
const expected = collectSync(xml);
expect(expected).toContainEqual([XmlEventType.CHARACTERS, "x\ny"]);
expect(expected).toContainEqual([XmlEventType.CDATA, "z\nw"]);
expect(expected).toContainEqual([
XmlEventType.START_ELEMENT, "r", "r", "", "",
[["a", "a", "", "", "1 2 3\r"]],
]);
expect(collectSync(encoder.encode(xml))).toEqual(expected);
expect(collectSync(chunks(xml))).toEqual(expected);
await expect(collectAsync(asyncChunks(xml))).resolves.toEqual(expected);
expect(Array.from(new EventReaderSync(xml)).map(eventToken)).toEqual(expected);
const events: unknown[][] = [];
for await (const event of new EventReader(asyncChunks(xml))) events.push(eventToken(event));
expect(events).toEqual(expected);
});
it("uses the configured fatal TextDecoder encoding for byte readers", async () => {
const xml = '<?xml version="1.0" encoding="UTF-16"?><root>한글</root>';
const expected = collectSync(xml);
Expand Down
11 changes: 11 additions & 0 deletions packages/stax-xml/test/w3c-canonical.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { describe, expect, it } from 'vitest';
import { canonicalXml } from '../scripts/w3c-canonical.js';
import { EventReaderSync } from '../src/sync/index.js';

describe('W3C canonical output', () => {
it('serializes normalized values, sorted attributes, CDATA, and processing instructions', () => {
expect(canonicalXml(new EventReaderSync('<?before data?><r z="&#13;" a="x\r\ny">a\r\nb<![CDATA[<c>]]><!--ignored--></r>')))
.toBe('<?before data?><r a="x y" z="&#13;">a&#10;b&lt;c&gt;</r>');
expect(canonicalXml(new EventReaderSync('<r>x\r\ny</r>'))).not.toBe('<r>x&#13;&#10;y</r>');
});
});
1 change: 1 addition & 0 deletions packages/stax-xml/vitest.unit.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export default defineConfig({
},
test: {
include: [
"test/w3c-canonical.test.ts",
"test/v1-reader-contract.test.ts",
"test/token-cursor-resume.test.ts",
"test/token-cursor-options-coverage.test.ts",
Expand Down
Loading