From d5e9a923ff9118c3a53a6e552642dbf23dd19aa1 Mon Sep 17 00:00:00 2001 From: Herbicide7665 Date: Wed, 21 Oct 2026 18:00:33 +0800 Subject: [PATCH 1/2] fix(input): sniff UTF-8/UTF-16LE/UTF-16BE BOMs in plan/steps/code/password readers Windows PowerShell 5.1's `>` redirection (the default shell on Windows 10/11) writes UTF-16LE with a BOM, and the CLI's own docs suggest exactly that redirection for creating plan files. Every input reader decoded UTF-8 unconditionally, so the CLI's own template failed its own lint. One shared reader (src/lib/text-file.ts) now sniffs the BOM and decodes accordingly; wired into the plan-from/plan-dir/plans-JSONL/steps/code and password-file readers. BOM-less UTF-8 files decode byte-for-byte unchanged. DOCUMENTATION.md notes the encoding behavior. Fixes #463. --- DOCUMENTATION.md | 6 ++++ package-lock.json | 30 ---------------- src/commands/project.ts | 3 +- src/commands/test.bom-input.spec.ts | 53 +++++++++++++++++++++++++++++ src/commands/test.ts | 22 ++++-------- src/lib/text-file.spec.ts | 43 +++++++++++++++++++++++ src/lib/text-file.ts | 35 +++++++++++++++++++ 7 files changed, 145 insertions(+), 47 deletions(-) create mode 100644 src/commands/test.bom-input.spec.ts create mode 100644 src/lib/text-file.spec.ts create mode 100644 src/lib/text-file.ts diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index da69607..f3ecd5e 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -404,6 +404,12 @@ testsprite test create --plan-template > plan.json testsprite test create --plan-from ./checkout.plan.json --dry-run --output json ``` +> **Windows PowerShell 5.1 note:** `>` in `powershell.exe` writes UTF-16LE with +> a BOM. The CLI sniffs UTF-8 / UTF-16LE / UTF-16BE BOMs on input, so plan +> files created this way work as-is; for other tools that expect UTF-8, write +> `Out-File -Encoding utf8` or use `pwsh` instead. + + `--plan-from` owns the complete test definition and does not support this setting; a supplied `--step-timeout` is warned about and ignored. Set it later with `test update --step-timeout `. #### `testsprite test create-batch` diff --git a/package-lock.json b/package-lock.json index bc6d78f..0efa6fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -794,9 +794,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -814,9 +811,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -834,9 +828,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -854,9 +845,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -874,9 +862,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -894,9 +879,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2517,9 +2499,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2541,9 +2520,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2565,9 +2541,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -2589,9 +2562,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/src/commands/project.ts b/src/commands/project.ts index b957130..2e5f8e3 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -13,6 +13,7 @@ import { import { resolveProfileName } from '../lib/config.js'; import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; import { firstSuiteNextSteps } from '../lib/first-suite-hint.js'; +import { readTextFileBomAware } from '../lib/text-file.js'; import type { FetchImpl, HttpClient } from '../lib/http.js'; import { globalShutdown, type ShutdownHandle } from '../lib/interrupt.js'; import { @@ -901,7 +902,7 @@ export async function runAutoAuth( // Resolve secrets from --*-file variants so they stay out of shell history. const password = opts.password ?? - (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); + (opts.passwordFile !== undefined ? readTextFileBomAware(opts.passwordFile).trim() : undefined); const clientSecret = opts.clientSecret ?? (opts.clientSecretFile !== undefined diff --git a/src/commands/test.bom-input.spec.ts b/src/commands/test.bom-input.spec.ts new file mode 100644 index 0000000..6eee5de --- /dev/null +++ b/src/commands/test.bom-input.spec.ts @@ -0,0 +1,53 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { runLint } from './test.js'; + +/** + * Integration tests for issue #463: the CLI's own docs suggest creating plan + * files with shell redirection, and Windows PowerShell 5.1's `>` writes + * UTF-16LE with a BOM. Every encoding the default Windows shell can produce + * must lint (and therefore create) cleanly. + */ +const PLAN = { + projectId: 'project_alice', + type: 'frontend', + name: 'checkout', + description: 'BOM encoding matrix', + planSteps: [{ type: 'action', description: 'navigate to home' }], +}; + +type BomEncoding = 'utf8' | 'utf8-bom' | 'utf16le' | 'utf16be'; + +function writeEncoded(dir: string, name: string, payload: string, enc: BomEncoding): string { + const path = join(dir, name); + let bytes: Buffer; + if (enc === 'utf8') { + bytes = Buffer.from(payload, 'utf8'); + } else if (enc === 'utf8-bom') { + bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(payload, 'utf8')]); + } else if (enc === 'utf16le') { + bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(payload, 'utf16le')]); + } else { + const body = Buffer.from(payload, 'utf16le'); + body.swap16(); + bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), body]); + } + writeFileSync(path, bytes); + return path; +} + +describe('plan files saved with a BOM (PowerShell 5.1 `>`) lint cleanly', () => { + for (const enc of ['utf16le', 'utf16be', 'utf8-bom', 'utf8'] as const) { + it(`lints a ${enc} plan file with zero issues`, async () => { + const dir = mkdtempSync(join(tmpdir(), 'bom-plan-')); + const path = writeEncoded(dir, `plan-${enc}.json`, JSON.stringify(PLAN, null, 2), enc); + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFrom: path }, + ); + expect(report.checked).toBe(1); + expect(report.issues).toEqual([]); + }); + } +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 8bf5ce5..b820f25 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -61,6 +61,7 @@ import { parseJUnitReportFormat, type JUnitTestResult, } from '../lib/junit-report.js'; +import { decodeWithBom, readTextFileBomAware } from '../lib/text-file.js'; import { ApiError, CLIError, @@ -1840,7 +1841,7 @@ function readCodeFileGuarded(path: string): string { function readCodeFile(path: string): string { try { - return stripBom(readFileSync(resolveAbsolute(path), 'utf8')); + return readTextFileBomAware(resolveAbsolute(path)); } catch (err) { const code = (err as NodeJS.ErrnoException).code; if (code === 'ENOENT') { @@ -1858,17 +1859,6 @@ function resolveAbsolute(path: string): string { return isAbsolute(path) ? path : resolve(process.cwd(), path); } -/** - * Drop a leading UTF-8 BOM (U+FEFF) from a freshly-read file. PowerShell 5.1's - * default `Set-Content -Encoding utf8` writes a BOM; without this strip, - * `JSON.parse` fails with an invisible "Unexpected token" error that renders - * as a blank character on most consoles. Most JSON parsers strip BOM at this - * boundary — we just bring this one in line. - */ -function stripBom(raw: string): string { - return raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw; -} - function requireNonEmpty(flagName: string, value: string): void { if (typeof value !== 'string' || value.trim().length === 0) { throw localValidationError(flagName, 'is required'); @@ -2130,7 +2120,7 @@ function parsePlanStepsFile(path: string): unknown { * that window, since a swap of the path afterward can no longer change what * gets read. * - * Returns `{ ok: true, raw }` on success (BOM already stripped) or + * Returns `{ ok: true, raw }` on success (UTF-8/UTF-16 BOM decoded) or * `{ ok: false, sizeBytes }` when the file is over `maxBytes` — the * oversize case is handed back rather than turned into an error here * because each caller throws its own `PAYLOAD_TOO_LARGE` envelope (distinct @@ -2231,7 +2221,7 @@ function readCappedFileGuarded( // A lower bound for a pipe or a growing file: reading stopped here. return { ok: false, sizeBytes: bytes.length }; } - return { ok: true, raw: stripBom(bytes.toString('utf8')) }; + return { ok: true, raw: decodeWithBom(bytes) }; } finally { closeSync(fd); } @@ -5192,7 +5182,7 @@ function readPlansFromDirGuarded(dir: string, stderrFn: (line: string) => void): let raw: string; try { - raw = stripBom(readFileSync(filePath, 'utf8')); + raw = readTextFileBomAware(filePath); } catch (err) { // Hard I/O error (permission denied etc.): re-throw so the user knows the // directory is unreadable — this is not a "skip" case. @@ -6099,7 +6089,7 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + for (const enc of ['utf8', 'utf8-bom', 'utf16le', 'utf16be'] as const) { + it(`decodes ${enc}`, () => { + expect(decodeWithBom(encoded(PAYLOAD, enc))).toBe(PAYLOAD); + }); + } + + it('passes BOM-less UTF-8 through unchanged', () => { + const raw = Buffer.from(PAYLOAD, 'utf8'); + expect(decodeWithBom(raw)).toBe(PAYLOAD); + expect(decodeWithBom(raw)).toBe(raw.toString('utf8')); + }); + + it('handles an empty buffer', () => { + expect(decodeWithBom(Buffer.alloc(0))).toBe(''); + }); +}); + +describe('readTextFileBomAware', () => { + it('reads a UTF-16LE file written by PowerShell 5.1 `>` back intact', () => { + const dir = mkdtempSync(join(tmpdir(), 'text-file-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, encoded(PAYLOAD, 'utf16le')); + expect(readTextFileBomAware(path)).toBe(PAYLOAD); + }); +}); diff --git a/src/lib/text-file.ts b/src/lib/text-file.ts new file mode 100644 index 0000000..aeb5f04 --- /dev/null +++ b/src/lib/text-file.ts @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs'; + +/** + * Decode a buffer that may start with a Unicode BOM: UTF-8 (EF BB BF), + * UTF-16LE (FF FE) or UTF-16BE (FE FF). BOM-less input decodes as UTF-8. + * + * Why: Windows PowerShell 5.1's `>` redirection (the default shell on + * Windows 10/11) writes UTF-16LE with a BOM, and this CLI's own documentation + * suggests exactly that redirection for creating plan files. Readers that + * decode UTF-8 unconditionally turn the CLI's own template into garbage. + */ +export function decodeWithBom(bytes: Buffer): string { + if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { + return bytes.subarray(3).toString('utf8'); + } + if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { + return bytes.subarray(2).toString('utf16le'); + } + if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { + // Buffers have no 'utf16be' decoder: swap the byte pairs, then decode LE. + const be = Buffer.from(bytes.subarray(2)); + be.swap16(); + return be.toString('utf16le'); + } + return bytes.toString('utf8'); +} + +/** + * Read a small text file, sniffing and stripping UTF-8 / UTF-16LE / UTF-16BE + * BOMs. BOM-less files decode as UTF-8, byte-for-byte unchanged from + * `readFileSync(path, 'utf8')`. + */ +export function readTextFileBomAware(path: string): string { + return decodeWithBom(readFileSync(path)); +} From 0d83eb906f7b6894633abeedfac7bb2166b7ac41 Mon Sep 17 00:00:00 2001 From: Herbicide7665 Date: Thu, 22 Oct 2026 09:42:45 +0800 Subject: [PATCH 2/2] fix(input): typed error for odd-byte UTF-16BE and password-file read failures CodeRabbit review follow-up: an odd trailing byte in UTF-16BE input made swap16() throw an untyped ERR_INVALID_BUFFER_SIZE that bypassed the capped readers' validation diagnostics. The decoder now rejects the incomplete code unit explicitly, readCappedFileGuarded maps decode failures to a typed localValidationError, and the project password-file reader gets the same guard. --- src/commands/project.ts | 11 +++++++-- src/commands/test.bom-input.spec.ts | 35 ++++++++++++++++++++++++++++- src/commands/test.ts | 9 +++++++- src/lib/text-file.ts | 8 ++++++- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/commands/project.ts b/src/commands/project.ts index 2e5f8e3..c284ae5 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -900,9 +900,16 @@ export async function runAutoAuth( } // Resolve secrets from --*-file variants so they stay out of shell history. + const readPasswordFile = (path: string): string => { + try { + return readTextFileBomAware(path).trim(); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError(`cannot read password file ${path}: ${reason}`); + } + }; const password = - opts.password ?? - (opts.passwordFile !== undefined ? readTextFileBomAware(opts.passwordFile).trim() : undefined); + opts.password ?? (opts.passwordFile !== undefined ? readPasswordFile(opts.passwordFile) : undefined); const clientSecret = opts.clientSecret ?? (opts.clientSecretFile !== undefined diff --git a/src/commands/test.bom-input.spec.ts b/src/commands/test.bom-input.spec.ts index 6eee5de..7f7abb9 100644 --- a/src/commands/test.bom-input.spec.ts +++ b/src/commands/test.bom-input.spec.ts @@ -2,7 +2,8 @@ import { mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { runLint } from './test.js'; +import { runCreateFromPlan, runLint } from './test.js'; +import { ApiError } from '../lib/errors.js'; /** * Integration tests for issue #463: the CLI's own docs suggest creating plan @@ -50,4 +51,36 @@ describe('plan files saved with a BOM (PowerShell 5.1 `>`) lint cleanly', () => expect(report.issues).toEqual([]); }); } + + it('an odd-byte UTF-16BE plan file fails create with the typed encoding error, not a crash', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bom-plan-')); + const body = Buffer.from(JSON.stringify(PLAN, null, 2), 'utf16le'); + body.swap16(); + const path = join(dir, 'plan-utf16be-odd.json'); + writeFileSync(path, Buffer.concat([Buffer.from([0xfe, 0xff]), body, Buffer.from([0x00])])); + writeFileSync( + join(dir, 'credentials'), + '[default]\napi_url = http://localhost:13502\napi_key = sk-user-test\n', + ); + const deps = { + credentialsPath: join(dir, 'credentials'), + fetchImpl: (() => { + throw new Error('unexpected network'); + }) as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }; + // Plan parsing fires before any network; the typed localValidationError + // (not a raw ERR_INVALID_BUFFER_SIZE) is what reaches the caller. The + // human-readable text rides on nextAction (envelope message is generic). + const err: unknown = await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: path }, + deps as never, + ).then( + () => null, + (e: unknown) => e, + ); + expect(err).toBeInstanceOf(ApiError); + expect(String((err as { nextAction?: string }).nextAction)).toContain('incomplete code unit'); + }); }); diff --git a/src/commands/test.ts b/src/commands/test.ts index b820f25..3f32248 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -2221,7 +2221,14 @@ function readCappedFileGuarded( // A lower bound for a pipe or a growing file: reading stopped here. return { ok: false, sizeBytes: bytes.length }; } - return { ok: true, raw: decodeWithBom(bytes) }; + let raw: string; + try { + raw = decodeWithBom(bytes); + } catch (err) { + const reason = err instanceof Error ? err.message : 'unknown error'; + throw localValidationError(field, `cannot decode ${path}: ${reason}`); + } + return { ok: true, raw }; } finally { closeSync(fd); } diff --git a/src/lib/text-file.ts b/src/lib/text-file.ts index aeb5f04..12ebe5c 100644 --- a/src/lib/text-file.ts +++ b/src/lib/text-file.ts @@ -18,7 +18,13 @@ export function decodeWithBom(bytes: Buffer): string { } if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { // Buffers have no 'utf16be' decoder: swap the byte pairs, then decode LE. - const be = Buffer.from(bytes.subarray(2)); + // An odd trailing byte is an incomplete code unit — reject it here instead + // of letting swap16() throw an untyped ERR_INVALID_BUFFER_SIZE. + const body = bytes.subarray(2); + if (body.length % 2 !== 0) { + throw new Error('UTF-16BE input ends with an incomplete code unit'); + } + const be = Buffer.from(body); be.swap16(); return be.toString('utf16le'); }