fix(input): sniff UTF-8/UTF-16LE/UTF-16BE BOMs in plan/steps/code/password readers

DOCUMENTATION.md

@@ -404,6 +404,12 @@ testsprite test create --plan-template > plan.json
 testsprite test create --plan-from ./checkout.plan.json --dry-run --output json
 ```
 
+> **Windows PowerShell 5.1 note:** `>` in `powershell.exe` writes UTF-16LE with
+> a BOM. The CLI sniffs UTF-8 / UTF-16LE / UTF-16BE BOMs on input, so plan
+> files created this way work as-is; for other tools that expect UTF-8, write
+> `Out-File -Encoding utf8` or use `pwsh` instead.
+
+
 `--plan-from` owns the complete test definition and does not support this setting; a supplied `--step-timeout` is warned about and ignored. Set it later with `test update --step-timeout <ms>`.
 
 #### `testsprite test create-batch`

src/commands/project.ts

@@ -13,6 +13,7 @@ import {
 import { resolveProfileName } from '../lib/config.js';
 import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js';
 import { firstSuiteNextSteps } from '../lib/first-suite-hint.js';
+import { readTextFileBomAware } from '../lib/text-file.js';
 import type { FetchImpl, HttpClient } from '../lib/http.js';
 import { globalShutdown, type ShutdownHandle } from '../lib/interrupt.js';
 import {
@@ -899,9 +900,16 @@ export async function runAutoAuth(
   }
 
   // Resolve secrets from --*-file variants so they stay out of shell history.
+  const readPasswordFile = (path: string): string => {
+    try {
+      return readTextFileBomAware(path).trim();
+    } catch (err) {
+      const reason = err instanceof Error ? err.message : 'unknown error';
+      throw localValidationError(`cannot read password file ${path}: ${reason}`);
+    }
+  };
   const password =
-    opts.password ??
-    (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined);
+    opts.password ?? (opts.passwordFile !== undefined ? readPasswordFile(opts.passwordFile) : undefined);
   const clientSecret =
     opts.clientSecret ??
     (opts.clientSecretFile !== undefined

src/commands/test.bom-input.spec.ts

@@ -0,0 +1,86 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { runCreateFromPlan, runLint } from './test.js';
+import { ApiError } from '../lib/errors.js';
+
+/**
+ * Integration tests for issue #463: the CLI's own docs suggest creating plan
+ * files with shell redirection, and Windows PowerShell 5.1's `>` writes
+ * UTF-16LE with a BOM. Every encoding the default Windows shell can produce
+ * must lint (and therefore create) cleanly.
+ */
+const PLAN = {
+  projectId: 'project_alice',
+  type: 'frontend',
+  name: 'checkout',
+  description: 'BOM encoding matrix',
+  planSteps: [{ type: 'action', description: 'navigate to home' }],
+};
+
+type BomEncoding = 'utf8' | 'utf8-bom' | 'utf16le' | 'utf16be';
+
+function writeEncoded(dir: string, name: string, payload: string, enc: BomEncoding): string {
+  const path = join(dir, name);
+  let bytes: Buffer;
+  if (enc === 'utf8') {
+    bytes = Buffer.from(payload, 'utf8');
+  } else if (enc === 'utf8-bom') {
+    bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(payload, 'utf8')]);
+  } else if (enc === 'utf16le') {
+    bytes = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(payload, 'utf16le')]);
+  } else {
+    const body = Buffer.from(payload, 'utf16le');
+    body.swap16();
+    bytes = Buffer.concat([Buffer.from([0xfe, 0xff]), body]);
+  }
+  writeFileSync(path, bytes);
+  return path;
+}
+
+describe('plan files saved with a BOM (PowerShell 5.1 `>`) lint cleanly', () => {
+  for (const enc of ['utf16le', 'utf16be', 'utf8-bom', 'utf8'] as const) {
+    it(`lints a ${enc} plan file with zero issues`, async () => {
+      const dir = mkdtempSync(join(tmpdir(), 'bom-plan-'));
+      const path = writeEncoded(dir, `plan-${enc}.json`, JSON.stringify(PLAN, null, 2), enc);
+      const report = await runLint(
+        { profile: 'default', output: 'json', debug: false, planFrom: path },
+      );
+      expect(report.checked).toBe(1);
+      expect(report.issues).toEqual([]);
+    });
+  }
+
+  it('an odd-byte UTF-16BE plan file fails create with the typed encoding error, not a crash', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'bom-plan-'));
+    const body = Buffer.from(JSON.stringify(PLAN, null, 2), 'utf16le');
+    body.swap16();
+    const path = join(dir, 'plan-utf16be-odd.json');
+    writeFileSync(path, Buffer.concat([Buffer.from([0xfe, 0xff]), body, Buffer.from([0x00])]));
+    writeFileSync(
+      join(dir, 'credentials'),
+      '[default]\napi_url = http://localhost:13502\napi_key = sk-user-test\n',
+    );
+    const deps = {
+      credentialsPath: join(dir, 'credentials'),
+      fetchImpl: (() => {
+        throw new Error('unexpected network');
+      }) as typeof fetch,
+      stdout: () => {},
+      stderr: () => {},
+    };
+    // Plan parsing fires before any network; the typed localValidationError
+    // (not a raw ERR_INVALID_BUFFER_SIZE) is what reaches the caller. The
+    // human-readable text rides on nextAction (envelope message is generic).
+    const err: unknown = await runCreateFromPlan(
+      { profile: 'default', output: 'json', debug: false, planFrom: path },
+      deps as never,
+    ).then(
+      () => null,
+      (e: unknown) => e,
+    );
+    expect(err).toBeInstanceOf(ApiError);
+    expect(String((err as { nextAction?: string }).nextAction)).toContain('incomplete code unit');
+  });
+});

src/commands/test.ts

@@ -61,6 +61,7 @@ import {
   parseJUnitReportFormat,
   type JUnitTestResult,
 } from '../lib/junit-report.js';
+import { decodeWithBom, readTextFileBomAware } from '../lib/text-file.js';
 import {
   ApiError,
   CLIError,
@@ -1840,7 +1841,7 @@ function readCodeFileGuarded(path: string): string {
 
 function readCodeFile(path: string): string {
   try {
-    return stripBom(readFileSync(resolveAbsolute(path), 'utf8'));
+    return readTextFileBomAware(resolveAbsolute(path));
   } catch (err) {
     const code = (err as NodeJS.ErrnoException).code;
     if (code === 'ENOENT') {
@@ -1858,17 +1859,6 @@ function resolveAbsolute(path: string): string {
   return isAbsolute(path) ? path : resolve(process.cwd(), path);
 }
 
-/**
- * Drop a leading UTF-8 BOM (U+FEFF) from a freshly-read file. PowerShell 5.1's
- * default `Set-Content -Encoding utf8` writes a BOM; without this strip,
- * `JSON.parse` fails with an invisible "Unexpected token" error that renders
- * as a blank character on most consoles. Most JSON parsers strip BOM at this
- * boundary — we just bring this one in line.
- */
-function stripBom(raw: string): string {
-  return raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw;
-}
-
 function requireNonEmpty(flagName: string, value: string): void {
   if (typeof value !== 'string' || value.trim().length === 0) {
     throw localValidationError(flagName, 'is required');
@@ -2130,7 +2120,7 @@ function parsePlanStepsFile(path: string): unknown {
  * that window, since a swap of the path afterward can no longer change what
  * gets read.
  *
- * Returns `{ ok: true, raw }` on success (BOM already stripped) or
+ * Returns `{ ok: true, raw }` on success (UTF-8/UTF-16 BOM decoded) or
  * `{ ok: false, sizeBytes }` when the file is over `maxBytes` — the
  * oversize case is handed back rather than turned into an error here
  * because each caller throws its own `PAYLOAD_TOO_LARGE` envelope (distinct
@@ -2231,7 +2221,14 @@ function readCappedFileGuarded(
       // A lower bound for a pipe or a growing file: reading stopped here.
       return { ok: false, sizeBytes: bytes.length };
     }
-    return { ok: true, raw: stripBom(bytes.toString('utf8')) };
+    let raw: string;
+    try {
+      raw = decodeWithBom(bytes);
+    } catch (err) {
+      const reason = err instanceof Error ? err.message : 'unknown error';
+      throw localValidationError(field, `cannot decode ${path}: ${reason}`);
+    }
+    return { ok: true, raw };
   } finally {
     closeSync(fd);
   }
@@ -5192,7 +5189,7 @@ function readPlansFromDirGuarded(dir: string, stderrFn: (line: string) => void):
 
     let raw: string;
     try {
-      raw = stripBom(readFileSync(filePath, 'utf8'));
+      raw = readTextFileBomAware(filePath);
     } catch (err) {
       // Hard I/O error (permission denied etc.): re-throw so the user knows the
       // directory is unreadable — this is not a "skip" case.
@@ -6099,7 +6096,7 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise<C
     const absolute = resolveAbsolute(opts.plans);
     let content: string;
     try {
-      content = readFileSync(absolute, 'utf8');
+      content = readTextFileBomAware(absolute);
     } catch {
       throw localValidationError('plans', `cannot read file: ${absolute}`);
     }

src/lib/text-file.spec.ts

@@ -0,0 +1,43 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { decodeWithBom, readTextFileBomAware } from './text-file.js';
+
+const PAYLOAD = '{"projectId":"project_alice","type":"frontend","name":"checkout"}';
+
+function encoded(payload: string, enc: 'utf8' | 'utf8-bom' | 'utf16le' | 'utf16be'): Buffer {
+  if (enc === 'utf8') return Buffer.from(payload, 'utf8');
+  if (enc === 'utf8-bom') return Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(payload, 'utf8')]);
+  if (enc === 'utf16le') return Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(payload, 'utf16le')]);
+  const body = Buffer.from(payload, 'utf16le');
+  body.swap16();
+  return Buffer.concat([Buffer.from([0xfe, 0xff]), body]);
+}
+
+describe('decodeWithBom', () => {
+  for (const enc of ['utf8', 'utf8-bom', 'utf16le', 'utf16be'] as const) {
+    it(`decodes ${enc}`, () => {
+      expect(decodeWithBom(encoded(PAYLOAD, enc))).toBe(PAYLOAD);
+    });
+  }
+
+  it('passes BOM-less UTF-8 through unchanged', () => {
+    const raw = Buffer.from(PAYLOAD, 'utf8');
+    expect(decodeWithBom(raw)).toBe(PAYLOAD);
+    expect(decodeWithBom(raw)).toBe(raw.toString('utf8'));
+  });
+
+  it('handles an empty buffer', () => {
+    expect(decodeWithBom(Buffer.alloc(0))).toBe('');
+  });
+});
+
+describe('readTextFileBomAware', () => {
+  it('reads a UTF-16LE file written by PowerShell 5.1 `>` back intact', () => {
+    const dir = mkdtempSync(join(tmpdir(), 'text-file-'));
+    const path = join(dir, 'plan.json');
+    writeFileSync(path, encoded(PAYLOAD, 'utf16le'));
+    expect(readTextFileBomAware(path)).toBe(PAYLOAD);
+  });
+});

src/lib/text-file.ts

@@ -0,0 +1,41 @@
+import { readFileSync } from 'node:fs';
+
+/**
+ * Decode a buffer that may start with a Unicode BOM: UTF-8 (EF BB BF),
+ * UTF-16LE (FF FE) or UTF-16BE (FE FF). BOM-less input decodes as UTF-8.
+ *
+ * Why: Windows PowerShell 5.1's `>` redirection (the default shell on
+ * Windows 10/11) writes UTF-16LE with a BOM, and this CLI's own documentation
+ * suggests exactly that redirection for creating plan files. Readers that
+ * decode UTF-8 unconditionally turn the CLI's own template into garbage.
+ */
+export function decodeWithBom(bytes: Buffer): string {
+  if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
+    return bytes.subarray(3).toString('utf8');
+  }
+  if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
+    return bytes.subarray(2).toString('utf16le');
+  }
+  if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+    // Buffers have no 'utf16be' decoder: swap the byte pairs, then decode LE.
+    // An odd trailing byte is an incomplete code unit — reject it here instead
+    // of letting swap16() throw an untyped ERR_INVALID_BUFFER_SIZE.
+    const body = bytes.subarray(2);
+    if (body.length % 2 !== 0) {
+      throw new Error('UTF-16BE input ends with an incomplete code unit');
+    }
+    const be = Buffer.from(body);
+    be.swap16();
+    return be.toString('utf16le');
+  }
+  return bytes.toString('utf8');
+}
+
+/**
+ * Read a small text file, sniffing and stripping UTF-8 / UTF-16LE / UTF-16BE
+ * BOMs. BOM-less files decode as UTF-8, byte-for-byte unchanged from
+ * `readFileSync(path, 'utf8')`.
+ */
+export function readTextFileBomAware(path: string): string {
+  return decodeWithBom(readFileSync(path));
+}