Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 173 additions & 14 deletions scripts/test-run-lock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,17 @@ import {
writeFileSync,
} from "node:fs";
import { hostname, tmpdir } from "node:os";
import { isAbsolute, join, posix, win32 } from "node:path";
import { join, posix, win32 } from "node:path";
import type { UserIdentity } from "../src/codex/convergence-types";
import {
resolveEffectiveUserIdentity,
resolveEffectiveUserRuntimeRoot,
} from "../src/codex/user-identity";
import { hardenSecretDir } from "../src/lib/windows-secret-acl";

export const TEST_RUN_ID_ENV = "OCX_TEST_RUN_ID";
export const TEST_RUN_LOCK_PATH_ENV = "OCX_TEST_RUN_LOCK_PATH";
export const TEST_RUN_LOCK_TOKEN_ENV = "OCX_TEST_RUN_LOCK_TOKEN";
export const TEST_RUN_NO_QUEUE_ENV = "OCX_TEST_NO_QUEUE";
const OWNER_FILE = "owner.json";
const MEMBERS_DIR = "members";
Expand All @@ -30,7 +38,7 @@ interface RuntimeDirectoryEntry {

export interface TestRunRuntimeFileSystem {
lstatSync(path: string): RuntimeDirectoryEntry;
mkdirSync(path: string, options: { mode: number }): void;
mkdirSync(path: string, options: { mode?: number; recursive?: boolean }): void;
accessSync(path: string, mode: number): void;
}

Expand All @@ -41,6 +49,9 @@ export interface ResolveDefaultTestRunLockPathOptions {
tempDir?: string;
hostName?: string;
fileSystem?: TestRunRuntimeFileSystem;
resolveIdentity?: () => UserIdentity;
resolveRuntimeRoot?: (identity: Extract<UserIdentity, { platform: "win32" }>) => string;
hardenWindowsDirectory?: (path: string) => void;
}

const runtimeFileSystem: TestRunRuntimeFileSystem = {
Expand All @@ -67,13 +78,32 @@ export interface AcquireTestRunLockOptions {
runId: string;
ownerPid?: number;
lockPath?: string;
validatedRuntimePath?: boolean;
joinExistingOwnerToken?: string;
pollMs?: number;
maxWaitMs?: number;
env?: NodeJS.ProcessEnv;
onWait?: (owner: TestRunLockOwner | null) => void;
onAcquiredAfterWait?: (elapsedMs: number) => void;
}

export interface ResolveInheritedTestRunLockOptions {
wrappedRunId?: string;
env?: NodeJS.ProcessEnv;
platform?: NodeJS.Platform;
hostName?: string;
}

export interface InheritedTestRunLock {
lockPath: string;
ownerToken: string;
}

export interface ResolveWrappedTestRunLockPathOptions {
env?: NodeJS.ProcessEnv;
resolve?: (options: ResolveDefaultTestRunLockPathOptions) => string;
}

export interface BareTestRunIdentity {
ownerPid: number;
runId: string;
Expand All @@ -89,11 +119,13 @@ function inspectRuntimeDirectory(options: {
fileSystem: TestRunRuntimeFileSystem;
expectedUid?: number;
requirePrivateMode?: boolean;
allowMissing?: boolean;
}): string | null {
let entry: RuntimeDirectoryEntry;
try {
entry = options.fileSystem.lstatSync(options.path);
} catch (error) {
if (options.allowMissing && errorCode(error) === "ENOENT") return null;
return `cannot be inspected (${errorCode(error)})`;
}
if (entry.isSymbolicLink() || !entry.isDirectory()) return "is not a real directory";
Expand All @@ -117,11 +149,55 @@ function machineDiscriminator(hostName: string): string {
return createHash("sha256").update(normalized).digest("hex").slice(0, 16);
}

/**
* Read a Windows-wrapper-provided lock path without repeating effective-user
* discovery in every Bun worker. Bare and POSIX runs never trust this environment
* value. Wrapped Windows paths are constrained to the exact host-specific lock
* filename and namespace shape; the wrapper remains responsible for resolving
* and validating the directory.
*/
export function resolveInheritedTestRunLock(
options: ResolveInheritedTestRunLockOptions,
): InheritedTestRunLock | undefined {
if (!options.wrappedRunId) return undefined;
const platform = options.platform ?? process.platform;
if (platform !== "win32") return undefined;
if (options.env?.[TEST_RUN_NO_QUEUE_ENV] === "1") return undefined;
const candidate = options.env?.[TEST_RUN_LOCK_PATH_ENV]?.trim();
const ownerToken = options.env?.[TEST_RUN_LOCK_TOKEN_ENV]?.trim();
if (!candidate || !ownerToken) {
throw new Error("The wrapped Bun test lock capability is incomplete; refusing inherited lock access.");
}

const expectedName = `opencodex-bun-test-${machineDiscriminator(options.hostName ?? hostname())}.lock`;
if (
!win32.isAbsolute(candidate)
|| win32.basename(candidate) !== expectedName
|| win32.basename(win32.dirname(candidate)) !== "bun-test-locks"
|| !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(ownerToken)
) {
throw new Error("The wrapped Bun test lock capability is invalid; refusing inherited lock access.");
}
return { lockPath: candidate, ownerToken };
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/** Resolve once in the wrapper, preserving the no-queue escape hatch as a true no-op. */
export function resolveWrappedTestRunLockPath(
options: ResolveWrappedTestRunLockPathOptions = {},
): string | undefined {
const env = options.env ?? process.env;
if (env[TEST_RUN_NO_QUEUE_ENV] === "1") return undefined;
return (options.resolve ?? resolveDefaultTestRunLockPath)({ env });
}

/**
* Resolve a user-scoped, machine-local default lock path without relying on HOME.
*
* Windows uses the effective-token SID and known-folder runtime root, then
* hardens a dedicated child with the repository's required ACL policy.
* POSIX XDG runtime directories are accepted only after an ownership and access
* check. The fallback is a private UID namespace under the OS temp directory.
* check, including exact mode 0700. The fallback is a private UID namespace
* under the OS temp directory.
* The hostname digest remains part of the lock name in either case: even if an
* administrator redirects either root to shared storage, host-local PID liveness
* checks can never reclaim or join another machine's lock.
Expand All @@ -144,18 +220,84 @@ export function resolveDefaultTestRunLockPath(
}

if (platform === "win32") {
if (!win32.isAbsolute(tempDir)) {
let identity: UserIdentity;
try {
identity = (options.resolveIdentity ?? resolveEffectiveUserIdentity)();
} catch (cause) {
throw new Error(
"Cannot resolve a safe user-scoped Bun test lock: the Windows OS temp/profile path is not absolute.",
"Cannot resolve a safe user-scoped Bun test lock: the Windows effective identity is unavailable.",
{ cause },
);
}
const issue = inspectRuntimeDirectory({ path: tempDir, fileSystem });
if (issue) {
if (identity.platform !== "win32") {
throw new Error(
`Cannot resolve a safe user-scoped Bun test lock: the Windows OS temp/profile directory ${issue}.`,
"Cannot resolve a safe user-scoped Bun test lock: the effective identity does not match Windows.",
);
}
return win32.join(tempDir, `opencodex-bun-test-${discriminator}.lock`);

let runtimeRoot: string;
try {
runtimeRoot = (options.resolveRuntimeRoot ?? resolveEffectiveUserRuntimeRoot)(identity);
} catch (cause) {
throw new Error(
"Cannot resolve a safe user-scoped Bun test lock: the Windows effective-user runtime is unavailable.",
{ cause },
);
}
if (!win32.isAbsolute(runtimeRoot)) {
throw new Error(
"Cannot resolve a safe user-scoped Bun test lock: the Windows effective-user runtime path is not absolute.",
);
}

const lockRoot = win32.join(runtimeRoot, "bun-test-locks");
const issueBeforeCreate = inspectRuntimeDirectory({
path: lockRoot,
fileSystem,
allowMissing: true,
});
if (issueBeforeCreate) {
throw new Error(
`Cannot resolve a safe user-scoped Bun test lock: the Windows lock directory ${issueBeforeCreate}.`,
);
}

try {
fileSystem.mkdirSync(lockRoot, { recursive: true });
} catch (cause) {
throw new Error(
"Cannot resolve a safe user-scoped Bun test lock: the Windows lock directory cannot be created.",
{ cause },
);
}
const issueBeforeHardening = inspectRuntimeDirectory({ path: lockRoot, fileSystem });
if (issueBeforeHardening) {
throw new Error(
`Cannot resolve a safe user-scoped Bun test lock: the Windows lock directory ${issueBeforeHardening}.`,
);
}
try {
const harden = options.hardenWindowsDirectory
?? ((path: string) => {
if (!hardenSecretDir(path, { required: true }).ok) {
throw new Error("required ACL hardening did not complete");
}
});
harden(lockRoot);
} catch (cause) {
throw new Error(
"Cannot resolve a safe user-scoped Bun test lock: the Windows lock directory cannot be secured.",
{ cause },
);
}

const issueAfterHardening = inspectRuntimeDirectory({ path: lockRoot, fileSystem });
if (issueAfterHardening) {
throw new Error(
`Cannot resolve a safe user-scoped Bun test lock: the Windows lock directory ${issueAfterHardening}.`,
);
}
return win32.join(lockRoot, `opencodex-bun-test-${discriminator}.lock`);
}

const uid = options.uid ?? (typeof process.getuid === "function" ? process.getuid() : undefined);
Expand All @@ -168,20 +310,21 @@ export function resolveDefaultTestRunLockPath(
const failures: string[] = [];
const xdgRuntimeDir = env.XDG_RUNTIME_DIR?.trim();
if (xdgRuntimeDir) {
if (!isAbsolute(xdgRuntimeDir)) {
if (!posix.isAbsolute(xdgRuntimeDir)) {
failures.push("XDG_RUNTIME_DIR is not absolute");
} else {
const issue = inspectRuntimeDirectory({
path: xdgRuntimeDir,
fileSystem,
expectedUid: uid,
requirePrivateMode: true,
});
if (!issue) return posix.join(xdgRuntimeDir, `opencodex-bun-test-${discriminator}.lock`);
failures.push(`XDG_RUNTIME_DIR ${issue}`);
}
}

if (!isAbsolute(tempDir)) {
if (!posix.isAbsolute(tempDir)) {
failures.push("the OS temporary directory is not absolute");
} else {
const fallback = posix.join(tempDir, `opencodex-test-runtime-${uid}`);
Expand All @@ -206,7 +349,7 @@ export function resolveDefaultTestRunLockPath(

throw new Error(
"Cannot resolve a safe user-scoped Bun test lock. "
+ "Ensure XDG_RUNTIME_DIR is an existing writable directory owned by the current uid, "
+ "Ensure XDG_RUNTIME_DIR is an existing writable mode-0700 directory owned by the current uid, "
+ `or make the OS temporary directory usable for a mode-0700 UID runtime (${failures.join("; ")}).`,
);
}
Expand Down Expand Up @@ -328,14 +471,30 @@ export async function acquireTestRunLock(options: AcquireTestRunLockOptions): Pr
return { acquired: false, owner: null, release() {} };
}

const usesDefaultLockPath = options.lockPath === undefined;
const usesDefaultLockPath = options.lockPath === undefined || options.validatedRuntimePath === true;
const lockPath = options.lockPath ?? resolveDefaultTestRunLockPath({ env });
const ownerPid = options.ownerPid ?? process.pid;
const pollMs = Math.max(1, options.pollMs ?? 5_000);
const maxWaitMs = Math.max(pollMs, options.maxWaitMs ?? 45 * 60 * 1000);
const startedAt = Date.now();
let announced = false;

if (options.joinExistingOwnerToken !== undefined) {
const current = readOwner(lockPath);
if (
current?.runId === options.runId
&& current.token === options.joinExistingOwnerToken
&& lockIsLive(lockPath, current)
&& registerMember(lockPath, current, process.pid)
) {
return { acquired: false, owner: current, release() {} };
}
throw new Error(
"Cannot join the wrapper-owned Bun test lock because its exact live owner no longer matches; "
+ "refusing to create or reclaim an inherited path.",
);
}

for (;;) {
const owner: TestRunLockOwner = {
version: 1,
Expand Down Expand Up @@ -367,7 +526,7 @@ export async function acquireTestRunLock(options: AcquireTestRunLockOptions): Pr
if (usesDefaultLockPath && ["EACCES", "ENOENT", "EPERM", "EROFS"].includes(code ?? "")) {
throw new Error(
"Cannot acquire the user-scoped Bun test lock because its validated runtime directory "
+ "became unavailable or unwritable. Check XDG_RUNTIME_DIR and the OS temporary directory.",
+ "became unavailable or unwritable. Check the effective-user runtime directory and its permissions.",
{ cause: error },
);
}
Expand Down
30 changes: 26 additions & 4 deletions scripts/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@ import { randomUUID } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
import { acquireTestRunLock, TEST_RUN_ID_ENV } from "./test-run-lock";
import {
acquireTestRunLock,
resolveWrappedTestRunLockPath,
TEST_RUN_ID_ENV,
TEST_RUN_LOCK_PATH_ENV,
TEST_RUN_LOCK_TOKEN_ENV,
} from "./test-run-lock";

export interface IsolatedTestEnvironment {
root: string;
Expand Down Expand Up @@ -383,8 +389,18 @@ function waitWithTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T |
});
}

async function runTestLane(lane: BunTestLane, runId: string, capture = false): Promise<{ exitCode: number; output: string }> {
const isolated = createIsolatedTestEnvironment({ ...process.env, [TEST_RUN_ID_ENV]: runId });
async function runTestLane(
lane: BunTestLane,
runId: string,
inheritedLock: { lockPath: string; ownerToken: string } | undefined,
capture = false,
): Promise<{ exitCode: number; output: string }> {
const isolated = createIsolatedTestEnvironment({
...process.env,
[TEST_RUN_ID_ENV]: runId,
[TEST_RUN_LOCK_PATH_ENV]: inheritedLock?.lockPath,
[TEST_RUN_LOCK_TOKEN_ENV]: inheritedLock?.ownerToken,
});
const startedAt = Date.now();
let interrupted: NodeJS.Signals | null = null;
const child = Bun.spawn([process.execPath, "test", ...lane.args], {
Expand Down Expand Up @@ -501,8 +517,11 @@ if (import.meta.main) {
);
}
const runId = randomUUID();
const lockPath = resolveWrappedTestRunLockPath({ env: process.env });
const lock = await acquireTestRunLock({
runId,
lockPath,
validatedRuntimePath: lockPath !== undefined,
onWait: owner => console.warn(
`[test] another Bun test run${owner ? ` (pid ${owner.pid})` : ""} holds the user lock; waiting. `
+ "Set OCX_TEST_NO_QUEUE=1 only for intentional overlap.",
Expand All @@ -511,10 +530,13 @@ if (import.meta.main) {
});
const startedAt = Date.now();
try {
const inheritedLock = process.platform === "win32" && lockPath && lock.owner
? { lockPath, ownerToken: lock.owner.token }
: undefined;
let exitCode = 0;
let captured = "";
for (const lane of resolveBunTestPlan(requestedTests, changedRun?.comparisonCommit)) {
const result = await runTestLane(lane, runId, Boolean(changedRun));
const result = await runTestLane(lane, runId, inheritedLock, Boolean(changedRun));
captured += result.output;
if (result.exitCode !== 0 && exitCode === 0) exitCode = result.exitCode;
if ([124, 130, 143].includes(result.exitCode)) break;
Expand Down
Loading
Loading