From 63c725270678928dbe8006fa0458b30c53627fef Mon Sep 17 00:00:00 2001 From: jobordu Date: Tue, 21 Apr 2026 20:13:37 +0100 Subject: [PATCH 1/4] feat(deployment): add shell-exec endpoint for synchronous command execution - Add POST /v1/deployments/{dseq}/leases/{gseq}/{oseq}/shell-exec endpoint - Use GranularAccess JWT with shell scope for authentication - WebSocket client for provider-proxy shell endpoint communication - CASL-based authorization with deployment ownership validation - 129 tests including adversarial hardening coverage Closes #3079 --- apps/api/package.json | 2 + .../shell-exec/shell-exec.controller.spec.ts | 213 +++ .../shell-exec/shell-exec.controller.ts | 71 + .../http-schemas/shell-exec.schema.ts | 24 + .../routes/shell-exec/shell-exec.router.ts | 59 + .../shell-exec/shell-exec.service.spec.ts | 1511 +++++++++++++++++ .../services/shell-exec/shell-exec.service.ts | 284 ++++ apps/api/src/routers/open-api-handlers.ts | 2 + apps/api/src/routes/deployment/index.ts | 3 +- package-lock.json | 2 + 10 files changed, 2170 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts create mode 100644 apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts create mode 100644 apps/api/src/deployment/http-schemas/shell-exec.schema.ts create mode 100644 apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts create mode 100644 apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts create mode 100644 apps/api/src/deployment/services/shell-exec/shell-exec.service.ts diff --git a/apps/api/package.json b/apps/api/package.json index 715dc23af5..cfe3165730 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -106,6 +106,7 @@ "ts-results": "^3.3.0", "tsyringe": "^4.10.0", "unleash-client": "^6.6.0", + "ws": "^8.18.2", "zod": "3.*" }, "devDependencies": { @@ -126,6 +127,7 @@ "@types/pg": "^8.11.6", "@types/semver": "^7.5.2", "@types/tar": "^6.1.13", + "@types/ws": "^8.5.4", "@typescript-eslint/eslint-plugin": "^7.12.0", "@vitest/coverage-v8": "^4.1.5", "drizzle-kit": "^0.22.7", diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts new file mode 100644 index 0000000000..d32b3080c2 --- /dev/null +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts @@ -0,0 +1,213 @@ +import { Err, Ok } from "ts-results"; +import { container } from "tsyringe"; +import { describe, expect, it } from "vitest"; +import { mock } from "vitest-mock-extended"; + +import { AuthService } from "@src/auth/services/auth.service"; +import type { WalletReaderService } from "@src/billing/services/wallet-reader/wallet-reader.service"; +import type { DeploymentReaderService } from "@src/deployment/services/deployment-reader/deployment-reader.service"; +import type { ShellExecService } from "@src/deployment/services/shell-exec/shell-exec.service"; +import type { ProviderService } from "@src/provider/services/provider/provider.service"; +import { ShellExecController } from "./shell-exec.controller"; + +import { createUser } from "@test/seeders/user.seeder"; + +describe(ShellExecController.name, () => { + it("throws 404 when deployment not found", async () => { + const { controller, deploymentReaderService } = setup(); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(undefined as never); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(404); + expect(error.message).toBe("Deployment not found"); + }); + + it("throws 404 when no lease matches the provided gseq and oseq", async () => { + const { controller } = setup(); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 99, oseq: 99, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(404); + expect(error.message).toBe("Lease not found"); + }); + + it("throws 500 when lease provider address is an empty string", async () => { + const { controller } = setup({ provider: "" }); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(500); + expect(error.message).toBe("Lease provider address not found"); + }); + + it("throws 400 when lease state is not active", async () => { + const { controller } = setup({ state: "closed" }); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(400); + expect(error.message).toBe("Lease is not active"); + }); + + it("throws 502 when shell exec service returns an error result", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("Command timed out")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Command execution timed out"); + }); + + it("returns the shell exec result on successful execution", async () => { + const { controller, shellExecService } = setup(); + + const result = await controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 }); + + expect(result).toEqual({ stdout: "output", stderr: "", exitCode: 0, truncated: false }); + expect(shellExecService.execute).toHaveBeenCalledWith({ + providerBaseUrl: "https://provider.example.com", + providerAddress: "akash1provider", + dseq: "1234", + gseq: 1, + oseq: 1, + service: "web", + command: "ls", + timeout: 60, + jwtToken: "test-token" + }); + }); + + it("throws 404 when provider info lookup returns null", async () => { + const { controller, providerService } = setup(); + providerService.getProvider.mockResolvedValue(null as never); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(404); + expect(error.message).toBe("Provider not found"); + }); + + it("throws 404 when deployment has an empty leases array", async () => { + const { controller, deploymentReaderService, deployment } = setup(); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue({ ...deployment, leases: [] } as never); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(404); + expect(error.message).toBe("Lease not found"); + }); + + it("finds the correct lease among multiple leases by gseq and oseq", async () => { + const { controller, deploymentReaderService, deployment, shellExecService } = setup(); + + const multiLeaseDeployment = { + ...deployment, + leases: [ + { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 1, oseq: 1 }, state: "active" }, + { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 2, oseq: 1, provider: "akash1provider2" }, state: "active" }, + { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 1, oseq: 2, provider: "akash1provider3" }, state: "active" } + ] + }; + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(multiLeaseDeployment as never); + + const result = await controller.exec({ dseq: "1234", gseq: 2, oseq: 1, command: "ls", service: "web", timeout: 60 }); + + expect(result).toEqual({ stdout: "output", stderr: "", exitCode: 0, truncated: false }); + expect(shellExecService.execute).toHaveBeenCalledWith(expect.objectContaining({ providerAddress: "akash1provider2" })); + }); + + it("throws 400 when deployment state is closed", async () => { + const { controller, deploymentReaderService, deployment } = setup(); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue({ ...deployment, deployment: { ...deployment.deployment, state: "closed" } } as never); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(400); + expect(error.message).toBe("Deployment is not active"); + }); + + it("throws 502 with stable message when WS connection fails", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("WebSocket connection failed: ECONNREFUSED")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Failed to connect to provider"); + }); + + it("throws 502 with stable message when provider returns error", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("Provider error: internal error")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Provider returned an error"); + }); + + async function captureError(fn: () => Promise): Promise { + try { + await fn(); + throw new Error("Expected function to throw"); + } catch (error) { + return error; + } + } + + function setup(overrides?: { provider?: string; state?: string }) { + const user = createUser(); + const deploymentReaderService = mock(); + const providerService = mock(); + const shellExecService = mock(); + const authService = mock({ currentUser: user }); + const walletReaderService = mock(); + + container.register(AuthService, { useValue: authService }); + + const controller = new ShellExecController(deploymentReaderService, providerService, shellExecService, authService, walletReaderService); + + const provider = overrides?.provider ?? "akash1provider"; + const state = overrides?.state ?? "active"; + + const deployment = { + deployment: { + id: { owner: "akash1owner", dseq: "1234" }, + state: "active", + hash: "abc123", + created_at: "12345" + }, + leases: [ + { + id: { owner: "akash1owner", dseq: "1234", gseq: 1, oseq: 1, provider, bseq: 0 }, + state, + price: { denom: "uakt", amount: "100" }, + created_at: "12345", + closed_on: "0", + status: null + } + ], + escrow_account: { + id: { scope: "deployment", xid: "1234" }, + state: { + owner: "akash1owner", + state: "open", + transferred: [], + settled_at: "12345", + funds: [{ denom: "uakt", amount: "1000" }], + deposits: [] + } + } + }; + + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(deployment as never); + walletReaderService.getWalletByUserId.mockResolvedValue({ id: 1, address: "akash1wallet" } as never); + providerService.toProviderAuth.mockResolvedValue({ type: "jwt" as const, token: "test-token" }); + providerService.getProvider.mockResolvedValue({ hostUri: "https://provider.example.com" } as never); + shellExecService.execute.mockResolvedValue(new Ok({ stdout: "output", stderr: "", exitCode: 0, truncated: false })); + + return { controller, deploymentReaderService, providerService, shellExecService, authService, walletReaderService, user, deployment }; + } +}); diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts new file mode 100644 index 0000000000..68b8adaa9d --- /dev/null +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts @@ -0,0 +1,71 @@ +import assert from "http-assert"; +import { singleton } from "tsyringe"; + +import { AuthService, Protected } from "@src/auth/services/auth.service"; +import { WalletReaderService } from "@src/billing/services/wallet-reader/wallet-reader.service"; +import { ShellExecRequest, ShellExecResponse } from "@src/deployment/http-schemas/shell-exec.schema"; +import { DeploymentReaderService } from "@src/deployment/services/deployment-reader/deployment-reader.service"; +import { ShellExecService } from "@src/deployment/services/shell-exec/shell-exec.service"; +import { ProviderService } from "@src/provider/services/provider/provider.service"; + +@singleton() +export class ShellExecController { + constructor( + private readonly deploymentReaderService: DeploymentReaderService, + private readonly providerService: ProviderService, + private readonly shellExecService: ShellExecService, + private readonly authService: AuthService, + private readonly walletReaderService: WalletReaderService + ) {} + + @Protected([{ action: "read", subject: "Lease" }]) + async exec(input: ShellExecRequest & { dseq: string; gseq: number; oseq: number }): Promise { + const userId = this.authService.currentUser.id; + + const deployment = await this.deploymentReaderService.findByUserIdAndDseq(userId, input.dseq); + + assert(deployment, 404, "Deployment not found"); + assert(deployment.deployment.state === "active", 400, "Deployment is not active"); + + const lease = deployment.leases.find(l => l.id.gseq === input.gseq && l.id.oseq === input.oseq); + + assert(lease, 404, "Lease not found"); + assert(lease.id.provider, 500, "Lease provider address not found"); + assert(lease.state === "active", 400, "Lease is not active"); + + const providerAddress = lease.id.provider; + + const wallet = await this.walletReaderService.getWalletByUserId(userId); + + const auth = await this.providerService.toProviderAuth({ walletId: wallet.id, provider: providerAddress }, ["shell"]); + + const providerInfo = await this.providerService.getProvider(providerAddress); + + assert(providerInfo, 404, "Provider not found"); + + const result = await this.shellExecService.execute({ + providerBaseUrl: providerInfo.hostUri, + providerAddress: providerAddress, + dseq: input.dseq, + gseq: input.gseq, + oseq: input.oseq, + service: input.service, + command: input.command, + timeout: input.timeout, + jwtToken: auth.token + }); + + if (!result.ok) { + const message = result.val.startsWith("Command timed out") + ? "Command execution timed out" + : result.val.startsWith("WebSocket connection failed") + ? "Failed to connect to provider" + : result.val.startsWith("Provider error") + ? "Provider returned an error" + : "Shell execution failed"; + assert(false, 502, message); + } + + return result.val; + } +} diff --git a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts new file mode 100644 index 0000000000..e1246af22d --- /dev/null +++ b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts @@ -0,0 +1,24 @@ +import { z } from "zod"; + +export const ShellExecParamsSchema = z.object({ + dseq: z.string(), + gseq: z.coerce.number().int().nonnegative(), + oseq: z.coerce.number().int().nonnegative() +}); + +export const ShellExecRequestSchema = z.object({ + command: z.string().min(1).max(4096), + service: z.string().min(1).max(253), + timeout: z.number().int().min(1).max(120).default(60) +}); + +export const ShellExecResponseSchema = z.object({ + stdout: z.string(), + stderr: z.string(), + exitCode: z.number(), + truncated: z.boolean() +}); + +export type ShellExecParams = z.infer; +export type ShellExecRequest = z.infer; +export type ShellExecResponse = z.infer; diff --git a/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts new file mode 100644 index 0000000000..8319a3901a --- /dev/null +++ b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts @@ -0,0 +1,59 @@ +import { container } from "tsyringe"; + +import { createRoute } from "@src/core/lib/create-route/create-route"; +import { OpenApiHonoHandler } from "@src/core/services/open-api-hono-handler/open-api-hono-handler"; +import { SECURITY_BEARER_OR_API_KEY } from "@src/core/services/openapi-docs/openapi-security"; +import { ShellExecController } from "@src/deployment/controllers/shell-exec/shell-exec.controller"; +import { ShellExecParamsSchema, ShellExecRequestSchema, ShellExecResponseSchema } from "@src/deployment/http-schemas/shell-exec.schema"; + +export const shellExecRouter = new OpenApiHonoHandler(); + +const shellExecRoute = createRoute({ + method: "post", + path: "/v1/deployments/{dseq}/leases/{gseq}/{oseq}/shell-exec", + summary: "Execute a shell command in a deployment container", + tags: ["Shell Exec"], + security: SECURITY_BEARER_OR_API_KEY, + request: { + params: ShellExecParamsSchema, + body: { + content: { + "application/json": { + schema: ShellExecRequestSchema + } + } + } + }, + responses: { + 200: { + description: "Command executed successfully", + content: { + "application/json": { + schema: ShellExecResponseSchema + } + } + }, + 400: { + description: "Invalid request (e.g., lease not active)" + }, + 401: { + description: "Unauthorized" + }, + 403: { + description: "Forbidden - user does not own this deployment" + }, + 404: { + description: "Deployment or lease not found" + }, + 502: { + description: "Provider proxy error" + } + } +}); + +shellExecRouter.openapi(shellExecRoute, async function routeShellExec(c) { + const params = c.req.valid("param"); + const body = c.req.valid("json"); + const result = await container.resolve(ShellExecController).exec({ ...params, ...body }); + return c.json(result, 200); +}); diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts new file mode 100644 index 0000000000..a92806ea45 --- /dev/null +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts @@ -0,0 +1,1511 @@ +import { faker } from "@faker-js/faker"; +import { describe, expect, it, vi } from "vitest"; +import { WebSocket } from "ws"; + +import { buildShellUrl, parseShellMessage, type ShellExecOutput, ShellExecService } from "./shell-exec.service"; + +vi.mock("ws", () => ({ + WebSocket: vi.fn() +})); + +describe(ShellExecService.name, () => { + describe("buildShellUrl", () => { + it("builds correct URL with single word command", () => { + const input = createShellExecInput({ command: "ls" }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/1/1/shell"); + expect(url).toContain("stdin=0"); + expect(url).toContain("tty=0"); + expect(url).toContain("podIndex=0"); + expect(url).toContain("service=test-service"); + expect(url).toContain("cmd0=ls"); + }); + + it("builds correct URL with multi-word command", () => { + const input = createShellExecInput({ command: "echo hello world" }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/1/1/shell"); + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).toContain("cmd2=world"); + }); + + it("builds correct URL with command containing path", () => { + const input = createShellExecInput({ command: "cat /run/secrets/db_password" }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/1/1/shell"); + expect(url).toContain("cmd0=cat"); + expect(url).toContain("cmd1=%2Frun%2Fsecrets%2Fdb_password"); + }); + + it("removes trailing slash from provider base URL", () => { + const input = createShellExecInput({ providerBaseUrl: "https://provider.example.com/" }); + + const url = buildShellUrl(input); + + expect(url).toMatch(/^https:\/\/provider\.example\.com\/lease/); + }); + + it("handles special characters in command", () => { + const input = createShellExecInput({ command: "echo hello&world" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello%26world"); + }); + + it("encodes service name with spaces", () => { + const input = createShellExecInput({ service: "my service" }); + + const url = buildShellUrl(input); + + expect(url).toContain("service=my%20service"); + }); + + it("builds URL with correct gseq and oseq", () => { + const input = createShellExecInput({ gseq: 3, oseq: 5 }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/3/5/shell"); + }); + + it("handles empty command string", () => { + const input = createShellExecInput({ command: "" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0="); + expect(url).not.toContain("cmd1="); + }); + + it("handles whitespace-only command string", () => { + const input = createShellExecInput({ command: " " }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0="); + expect(url).not.toContain("cmd1="); + }); + + it("handles command with consecutive spaces", () => { + const input = createShellExecInput({ command: "echo hello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + }); + + it("whitespace-only command should not produce multiple cmd params", () => { + const input = createShellExecInput({ command: " " }); + + const url = buildShellUrl(input); + + expect(url).not.toContain("cmd1="); + expect(url).not.toContain("cmd2="); + expect(url).not.toContain("cmd3="); + }); + + it("treats newline as a token delimiter", () => { + const input = createShellExecInput({ command: "echo\nhello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).not.toContain("cmd2="); + }); + + it("command with leading spaces should filter them out", () => { + const input = createShellExecInput({ command: " echo hello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).not.toContain("cmd2="); + }); + + it("treats tab as a token delimiter", () => { + const input = createShellExecInput({ command: "echo\thello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).not.toContain("cmd2="); + }); + + it("collapses mixed whitespace (spaces and tabs) between tokens", () => { + const input = createShellExecInput({ command: "echo \t hello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).not.toContain("cmd2="); + }); + + it("splits on multiple consecutive tabs", () => { + const input = createShellExecInput({ command: "cmd\t\targ" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=cmd"); + expect(url).toContain("cmd1=arg"); + expect(url).not.toContain("cmd2="); + }); + + it("keeps a double-quoted argument with spaces as a single token", () => { + const input = createShellExecInput({ command: 'echo "hello world"' }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello%20world"); + expect(url).not.toContain("cmd2="); + }); + + it("keeps a single-quoted argument with spaces as a single token", () => { + const input = createShellExecInput({ command: "echo 'a b c'" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=a%20b%20c"); + expect(url).not.toContain("cmd2="); + }); + + it("preserves the post-deploy secret-injection command as argv (sh -c ...)", () => { + const input = createShellExecInput({ command: 'sh -c "echo SECRET=v > /run/secrets/.env"' }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=sh"); + expect(url).toContain("cmd1=-c"); + expect(url).toContain("cmd2=echo%20SECRET%3Dv%20%3E%20%2Frun%2Fsecrets%2F.env"); + expect(url).not.toContain("cmd3="); + }); + + it("treats a backslash-escaped space as part of the token", () => { + const input = createShellExecInput({ command: "echo a\\ b" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=a%20b"); + expect(url).not.toContain("cmd2="); + }); + + it("handles service name with hash fragment character", () => { + const input = createShellExecInput({ service: "svc#1" }); + + const url = buildShellUrl(input); + + expect(url).toContain("service=svc%231"); + }); + + it("handles service name with question mark", () => { + const input = createShellExecInput({ service: "my?service" }); + + const url = buildShellUrl(input); + + expect(url).toContain("service=my%3Fservice"); + }); + + it("handles provider base URL without trailing slash", () => { + const input = createShellExecInput({ providerBaseUrl: "https://provider.example.com" }); + + const url = buildShellUrl(input); + + expect(url).toMatch(/^https:\/\/provider\.example\.com\/lease/); + }); + + it("command with carriage return should be URL-encoded", () => { + const input = createShellExecInput({ command: "echo\rhello" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo%0Dhello"); + expect(url).not.toContain("cmd1="); + }); + + it("handles Unicode characters in command", () => { + const input = createShellExecInput({ command: "echo 日本語" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=%E6%97%A5%E6%9C%AC%E8%AA%9E"); + }); + + it("handles gseq of 0 in URL path", () => { + const input = createShellExecInput({ gseq: 0 }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/0/1/shell"); + }); + + it("handles oseq of 0 in URL path", () => { + const input = createShellExecInput({ oseq: 0 }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/1/0/shell"); + }); + + it("builds URL with empty dseq (all slashes preserved)", () => { + const input = createShellExecInput({ dseq: "" }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease//1/1/shell"); + }); + + it("builds URL with empty service name", () => { + const input = createShellExecInput({ service: "" }); + + const url = buildShellUrl(input); + + expect(url).toContain("service="); + }); + + it("builds URL with empty providerBaseUrl (should not produce double slash)", () => { + const input = createShellExecInput({ providerBaseUrl: "" }); + + const url = buildShellUrl(input); + + expect(url).toMatch(/^\/lease/); + }); + }); + + describe("binary message stream marker edge cases", () => { + it("binary message with stream marker 3 (unexpected) should not corrupt state", () => { + const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("binary message with stream marker 255 (unexpected) should not corrupt state", () => { + const result = parseShellMessage("[255, 72]"); + + expect(result).toBeNull(); + }); + }); + + describe("parseShellMessage", () => { + it("should return exit_code parsed from JSON message", () => { + const result = parseShellMessage('{"exit_code": 42}'); + + expect(result).toEqual({ type: "exit_code", exit_code: 42 }); + }); + + it("should return data with stream from JSON message containing message field", () => { + const result = parseShellMessage('{"message": "hello"}'); + + expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); + }); + + it("should return null for non-JSON string message", () => { + const result = parseShellMessage("plain text"); + + expect(result).toBeNull(); + }); + }); + + describe("parseShellMessage edge cases", () => { + it("returns null for empty JSON object", () => { + const result = parseShellMessage("{}"); + + expect(result).toBeNull(); + }); + + it("returns null for JSON with null values", () => { + const result = parseShellMessage('{"exit_code": null, "message": null}'); + + expect(result).toBeNull(); + }); + + it("prefers exit_code over message when both present", () => { + const result = parseShellMessage('{"exit_code": 0, "message": "ignored"}'); + + expect(result).toEqual({ type: "exit_code", exit_code: 0 }); + }); + + it("returns null for JSON with undefined values", () => { + const result = parseShellMessage('{"exit_code": undefined}'); + + expect(result).toBeNull(); + }); + + it("returns null for malformed JSON", () => { + const result = parseShellMessage('{"exit_code": }'); + + expect(result).toBeNull(); + }); + + it("returns null when exit_code is string instead of number", () => { + const result = parseShellMessage('{"exit_code": "42"}'); + + expect(result).toBeNull(); + }); + + it("returns null when exit_code is boolean true", () => { + const result = parseShellMessage('{"exit_code": true}'); + + expect(result).toBeNull(); + }); + + it("returns null when exit_code is boolean false", () => { + const result = parseShellMessage('{"exit_code": false}'); + + expect(result).toBeNull(); + }); + + it("returns null when message is number instead of string", () => { + const result = parseShellMessage('{"message": 123}'); + + expect(result).toBeNull(); + }); + + it("returns null for array input masquerading as JSON object", () => { + const result = parseShellMessage("[]"); + + expect(result).toBeNull(); + }); + + it("returns null for string input that looks like JSON but isn't", () => { + const result = parseShellMessage('"not an object"'); + + expect(result).toBeNull(); + }); + + it("returns null for number input", () => { + const result = parseShellMessage("123"); + + expect(result).toBeNull(); + }); + + it("returns null for boolean JSON", () => { + const result = parseShellMessage("true"); + + expect(result).toBeNull(); + }); + }); + + describe("binary message parsing edge cases", () => { + it("ignores empty binary array message", () => { + const result = parseShellMessage("[]"); + + expect(result).toBeNull(); + }); + + it("ignores binary message with only stream marker and no payload", () => { + const result = parseShellMessage("[1]"); + + expect(result).toBeNull(); + }); + + it("ignores binary message with invalid stream marker byte", () => { + const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("handles exit_code 0 as valid number", () => { + const result = parseShellMessage('{"exit_code": 0}'); + + expect(result).toEqual({ type: "exit_code", exit_code: 0 }); + }); + + it("handles negative exit_code", () => { + const result = parseShellMessage('{"exit_code": -1}'); + + expect(result).toEqual({ type: "exit_code", exit_code: -1 }); + }); + + it("returns null for exit_code as NaN", () => { + const result = parseShellMessage('{"exit_code": NaN}'); + + expect(result).toBeNull(); + }); + + it("returns null for exit_code as Infinity", () => { + const result = parseShellMessage('{"exit_code": Infinity}'); + + expect(result).toBeNull(); + }); + + it("returns null for exit_code as -Infinity", () => { + const result = parseShellMessage('{"exit_code": -Infinity}'); + + expect(result).toBeNull(); + }); + + it("returns null for message as empty string", () => { + const result = parseShellMessage('{"message": ""}'); + + expect(result).toBeNull(); + }); + + it("handles very large exit_code number", () => { + const result = parseShellMessage('{"exit_code": 9999999999}'); + + expect(result).toEqual({ type: "exit_code", exit_code: 9999999999 }); + }); + + it("should fall back to message when exit_code is non-numeric string", () => { + const result = parseShellMessage('{"exit_code": "invalid", "message": "hello"}'); + + expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); + }); + + it("should fall back to message when exit_code is object", () => { + const result = parseShellMessage('{"exit_code": {}, "message": "hello"}'); + + expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); + }); + + it("returns null for JSON starting with whitespace before brace", () => { + const result = parseShellMessage(' {"exit_code": 0}'); + + expect(result).toBeNull(); + }); + }); + + describe("binary message edge cases", () => { + it("should handle exit_code 0 in JSON message", () => { + // JSON messages with exit_code are handled by parseShellMessage + const result = parseShellMessage('{"exit_code": 0}'); + + expect(result).toEqual({ type: "exit_code", exit_code: 0 }); + }); + + it("should return null for binary array with unexpected stream marker byte", () => { + // Stream marker 3 is not valid (only 1=stdout, 2=stderr are valid) + const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("should return null for empty binary array", () => { + const result = parseShellMessage("[]"); + + expect(result).toBeNull(); + }); + }); + + describe("buildShellUrl edge cases", () => { + it("treats an unquoted backslash as a shell escape (removes it)", () => { + const input = createShellExecInput({ command: "echo \\hello\\world" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=helloworld"); + }); + + it("preserves literal backslashes inside single quotes", () => { + const input = createShellExecInput({ command: "echo '\\hello\\world'" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=%5Chello%5Cworld"); + }); + + it("URL-encodes dseq containing forward slash to prevent path traversal", () => { + const input = createShellExecInput({ dseq: "foo/bar/baz" }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/foo%2Fbar%2Fbaz/"); + }); + + it("handles negative gseq by including it in URL path (no validation)", () => { + const input = createShellExecInput({ gseq: -1 }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/-1/1/shell"); + }); + + it("handles negative oseq by including it in URL path (no validation)", () => { + const input = createShellExecInput({ oseq: -5 }); + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/test-dseq/1/-5/shell"); + }); + + it("command consisting only of tabs is trimmed to empty (tabs are whitespace)", () => { + const input = createShellExecInput({ command: "\t\t" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0="); + expect(url).not.toContain("cmd1="); + }); + + it("command with only newlines and tabs is trimmed to empty like tabs (all Unicode whitespace trimmed by JS trim)", () => { + const input = createShellExecInput({ command: "\n\t\n" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0="); + expect(url).not.toContain("cmd1="); + }); + }); + + describe("binary single-byte exit code edge cases", () => { + it("binary message [0] as single byte exit code should set exitCode to 0", () => { + const result = parseShellMessage("[0]"); + + expect(result).toBeNull(); + }); + + it("binary message with stream marker 1 and no payload should be ignored", () => { + const result = parseShellMessage("[1]"); + + expect(result).toBeNull(); + }); + + it("binary message [0, 72] should handle exit code 0 via firstByte check", () => { + const result = parseShellMessage("[0, 72]"); + + expect(result).toBeNull(); + }); + + it("binary message [0, 72, 101, 108, 108, 111] with stream marker 0 AND payload bytes should be ignored (exit code 0 followed by data is not valid)", () => { + const result = parseShellMessage("[0, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("NaN exit_code should be rejected (typeof NaN === 'number' in JS)", () => { + const result = parseShellMessage('{"exit_code": NaN}'); + + expect(result).toBeNull(); + }); + }); + + describe("parseShellMessage edge cases", () => { + it("returns null for JSON string null literal", () => { + const result = parseShellMessage("null"); + + expect(result).toBeNull(); + }); + + it("returns null for empty string input", () => { + const result = parseShellMessage(""); + + expect(result).toBeNull(); + }); + }); + + function createShellExecInput( + overrides?: Partial<{ command: string; timeout: number; providerBaseUrl: string; gseq: number; oseq: number; service: string; dseq: string }> + ): Parameters[0] { + return { + providerBaseUrl: overrides?.providerBaseUrl ?? "https://provider.example.com", + providerAddress: faker.string.alphanumeric(44), + dseq: overrides?.dseq ?? "test-dseq", + gseq: overrides?.gseq ?? 1, + oseq: overrides?.oseq ?? 1, + service: overrides?.service ?? "test-service", + command: overrides?.command ?? "echo Hello", + timeout: overrides?.timeout ?? 60, + jwtToken: faker.string.alphanumeric(100) + }; + } + + describe("buildShellUrl edge cases - undefined/null providerBaseUrl", () => { + it("throws TypeError when providerBaseUrl is undefined (replace on undefined)", () => { + const input = { + providerBaseUrl: undefined as any, + dseq: "test-dseq", + gseq: 1, + oseq: 1, + service: "test-service", + command: "echo hello" + }; + + expect(() => buildShellUrl(input)).toThrow(TypeError); + }); + + it("throws TypeError when providerBaseUrl is null (replace on null)", () => { + const input = { + providerBaseUrl: null as any, + dseq: "test-dseq", + gseq: 1, + oseq: 1, + service: "test-service", + command: "echo hello" + }; + + expect(() => buildShellUrl(input)).toThrow(TypeError); + }); + }); + + describe("buildShellUrl edge cases - undefined inputs", () => { + it("throws TypeError when command is undefined (trim on undefined)", () => { + const input = { + providerBaseUrl: "https://provider.example.com", + dseq: "test-dseq", + gseq: 1, + oseq: 1, + service: "test-service", + command: undefined as any + }; + + expect(() => buildShellUrl(input)).toThrow(TypeError); + }); + + it("throws TypeError when command is null (trim on null)", () => { + const input = { + providerBaseUrl: "https://provider.example.com", + dseq: "test-dseq", + gseq: 1, + oseq: 1, + service: "test-service", + command: null as any + }; + + expect(() => buildShellUrl(input)).toThrow(TypeError); + }); + + it("encodes undefined dseq as literal string 'undefined' in URL path", () => { + const input = { + providerBaseUrl: "https://provider.example.com", + dseq: undefined as any, + gseq: 1, + oseq: 1, + service: "test-service", + command: "echo hello" + }; + + const url = buildShellUrl(input); + + expect(url).toContain("/lease/undefined/1/1/shell"); + }); + + it("encodes undefined service as literal string 'undefined'", () => { + const input = { + providerBaseUrl: "https://provider.example.com", + dseq: "test-dseq", + gseq: 1, + oseq: 1, + service: undefined as any, + command: "echo hello" + }; + + const url = buildShellUrl(input); + + expect(url).toContain("service=undefined"); + }); + }); + + describe("parseShellMessage edge cases - null and object message types", () => { + it("returns null when message.message is null (binary path)", () => { + const result = parseShellMessage("[1, null]"); + + expect(result).toBeNull(); + }); + + it("returns null when message.message is an object instead of string/array", () => { + const result = parseShellMessage('{"type": "websocket", "message": {}}'); + + expect(result).toBeNull(); + }); + + it("returns null when message.message is a number instead of string/array", () => { + const result = parseShellMessage('{"type": "websocket", "message": 123}'); + + expect(result).toBeNull(); + }); + }); + + describe("binary message parsing - stream marker 0 with payload", () => { + it("binary message [0, 72, 101] with stream marker 0 AND payload bytes should be ignored (exit code not set)", () => { + const result = parseShellMessage("[0, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("binary message [0, 0] with stream marker 0 AND null byte should be ignored", () => { + const result = parseShellMessage("[0, 0]"); + + expect(result).toBeNull(); + }); + + it("binary message [1, 256] with out-of-range byte should not crash (replacement character)", () => { + const result = parseShellMessage("[1, 256]"); + + expect(result).toBeNull(); + }); + + it("binary message [1, -1] with negative byte value should not crash", () => { + const result = parseShellMessage("[1, -1]"); + + expect(result).toBeNull(); + }); + }); + + describe("parseShellMessage - whitespace-only message string", () => { + it("returns data for message with only spaces (length > 0 check passes)", () => { + const result = parseShellMessage('{"message": " "}'); + + expect(result).toEqual({ type: "data", data: " ", stream: "stdout" }); + }); + + it("returns data for message with only tabs", () => { + const result = parseShellMessage('{"message": "\\t\\t"}'); + + expect(result).toEqual({ type: "data", data: "\t\t", stream: "stdout" }); + }); + + it("returns data for message containing only newlines (whitespace IS valid shell output)", () => { + const result = parseShellMessage('{"message": "\\n\\n\\n"}'); + + expect(result).toEqual({ type: "data", data: "\n\n\n", stream: "stdout" }); + }); + + it("returns data for message containing only carriage returns", () => { + const result = parseShellMessage('{"message": "\\r\\r\\r"}'); + + expect(result).toEqual({ type: "data", data: "\r\r\r", stream: "stdout" }); + }); + + it("returns data for message containing mixed whitespace (space, newline, tab)", () => { + const result = parseShellMessage('{"message": " \\n\\t "}'); + + expect(result).toEqual({ type: "data", data: " \n\t ", stream: "stdout" }); + }); + }); + + describe("parseShellMessage - binary stderr stream marker 2", () => { + it("binary message [2, 72, 101, 108, 108, 111] with stderr stream marker returns null (binary path)", () => { + const result = parseShellMessage("[2, 72, 101, 108, 108, 111]"); + + expect(result).toBeNull(); + }); + + it("binary message [2] with stderr stream marker and no payload returns null", () => { + const result = parseShellMessage("[2]"); + + expect(result).toBeNull(); + }); + }); + + describe("execute - message.error field edge cases", () => { + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + if (!handlers[event]) handlers[event] = []; + handlers[event].push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: any[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } + + it("message.error truthy string returns Err with provider error", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "some data"}' }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "error", error: "provider connection failed" }))); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("Provider error: provider connection failed"); + }); + + it("WebSocket error event returns Err with connection failure message", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "partial"}' }))); + mockWs._trigger("error", new Error("ECONNREFUSED")); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("WebSocket connection failed: ECONNREFUSED"); + }); + }); + + describe("execute - timeout + close double resolve", () => { + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + if (!handlers[event]) handlers[event] = []; + handlers[event].push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: any[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } + + it("timeout resolving then close event firing causes double resolve (first Err should win)", async () => { + vi.useFakeTimers(); + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: 5 })); + + mockWs._trigger("open"); + + vi.advanceTimersByTime(5000); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Command timed out"); + + vi.useRealTimers(); + }); + + it("negative timeout fires immediately returning timeout error", async () => { + vi.useFakeTimers(); + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: -1 })); + + mockWs._trigger("open"); + + vi.advanceTimersByTime(0); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Command timed out"); + + vi.useRealTimers(); + }); + }); + + describe("execute - string message parse failure silently dropped", () => { + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + if (!handlers[event]) handlers[event] = []; + handlers[event].push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: any[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } + + it("string message that fails parseShellMessage is silently dropped (not accumulated)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "first" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: "this is not JSON so parseShellMessage returns null" }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "third" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("firstthird"); + }); + }); + + describe("execute - closed message after exitCode already set", () => { + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + if (!handlers[event]) handlers[event] = []; + handlers[event].push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: any[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } + + it("message.closed true arrives after exit_code 0 - exitCode stays 0 (first exit_code wins)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + }); + + describe("buildShellUrl - command with multiple consecutive spaces", () => { + it("command with three consecutive spaces between words splits correctly", () => { + const input = createShellExecInput({ command: "echo hello world" }); + + const url = buildShellUrl(input); + + expect(url).toContain("cmd0=echo"); + expect(url).toContain("cmd1=hello"); + expect(url).toContain("cmd2=world"); + expect(url).not.toContain("cmd3="); + }); + }); + + describe("execute - WebSocket integration edge cases", () => { + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: any[]) => void) => { + if (!handlers[event]) handlers[event] = []; + handlers[event].push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: any[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } + + it("discards binary payload with stream marker 0 instead of routing to stdout or stderr", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: [0, 72, 101, 108, 108, 111] + }) + ) + ); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe(""); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("allows combined binary stdout+stderr to exceed MAX_OUTPUT_SIZE by checking each stream independently", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + + const halfSize = 524288; + const stdoutData = new Array(halfSize + 1).fill(65); + stdoutData[0] = 1; + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: stdoutData }))); + + const stderrData = new Array(halfSize + 2).fill(66); + stderrData[0] = 2; + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: stderrData }))); + + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [0] }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).truncated).toBe(true); + }); + + it("binary stream marker 1 (stdout) appends data to stdout and combines with string messages", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "string-" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [1, 66, 105, 110, 97, 114, 121] }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("string-Binary"); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("unknown message type does not corrupt stdout accumulation", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "first" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "unknown", data: "corrupt" }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "second" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("firstsecond"); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("binary message with stream marker 1 and no payload is ignored (length check)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "data" }) }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [1] }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("data"); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("leaves promise unresolved when error event fires without close event", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: 30 })); + + mockWs._trigger("error", new Error("connection failed")); + + const raceResult = await Promise.race([ + promise.then(r => ({ status: "resolved" as const, result: r })), + new Promise<{ status: "pending" }>(resolve => setTimeout(() => resolve({ status: "pending" }), 100)) + ]); + + mockWs._trigger("close"); + + expect(raceResult.status).toBe("resolved"); + }); + + it("error event resolves before close event can fire (first resolve wins)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("error", new Error("connection failed")); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("WebSocket connection failed: connection failed"); + }); + + it("discards accumulated stdout data when timeout fires returning only error string", async () => { + vi.useFakeTimers(); + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: 5 })); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "partial output"}' }))); + + vi.advanceTimersByTime(5000); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Command timed out"); + + vi.useRealTimers(); + }); + + it("defaults exitCode to 1 when message.closed arrives with no prior exit_code", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(1); + }); + + it("uses last exit_code when multiple exit_code JSON messages arrive sequentially", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 42}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(42); + }); + + it("string message path falls back to message when exit_code is non-numeric string", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const shellExecService = new ShellExecService(); + const promise = shellExecService.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ exit_code: "invalid", message: "fallback data" }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("fallback data"); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("resolves with timeout error immediately when timeout is 0 (setTimeout 0ms)", async () => { + vi.useFakeTimers(); + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: 0 })); + + vi.advanceTimersByTime(0); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Command timed out"); + expect(mockWs.close).toHaveBeenCalled(); + + vi.useRealTimers(); + }); + + it("resolves with error when WebSocket closes without ever receiving exit code", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Connection closed without exit code"); + }); + + it("message arriving before timeout does not cause double resolve when timeout fires and close event also fires", async () => { + vi.useFakeTimers(); + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput({ timeout: 5 })); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + + vi.advanceTimersByTime(5000); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + + vi.useRealTimers(); + }); + + it("message.closed true followed by close event does not double-resolve (closed sets exitCode=1, close resolves)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(1); + }); + + it("string message path never routes to stderr even with error-like content (parseShellMessage always returns stream: stdout)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ message: "error: something went wrong" }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("error: something went wrong"); + expect((result.val as ShellExecOutput).stderr).toBe(""); + }); + + it("binary payload with firstByte 3-255 is silently discarded (dead zone in protocol)", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: [3, 72, 101, 108, 108, 111] + }) + ) + ); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: [255, 87, 111, 114, 108, 100] + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [0] }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe(""); + expect((result.val as ShellExecOutput).stderr).toBe(""); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + + it("output at exactly MAX_OUTPUT_SIZE (1048576 bytes) is NOT truncated", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + const exactSize = 1024 * 1024; + const data = "A".repeat(exactSize); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ message: data }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).truncated).toBe(false); + expect((result.val as ShellExecOutput).stdout).toBe(data); + expect((result.val as ShellExecOutput).stdout.length).toBe(1048576); + }); + + it("output exceeding MAX_OUTPUT_SIZE by 1 byte (1048577) IS truncated and data is discarded", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + const overSize = 1024 * 1024 + 1; + const data = "A".repeat(overSize); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ message: data }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).truncated).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe(""); + }); + + it("pong messages interspersed with data messages do not corrupt output accumulation", async () => { + const mockWs = createMockWebSocket(); + vi.mocked(WebSocket).mockImplementation(function (this: any) { + return mockWs; + }); + + const service = new ShellExecService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ message: "Hello " }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "pong" }))); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: JSON.stringify({ message: "World" }) + }) + ) + ); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); + mockWs._trigger("close"); + + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("Hello World"); + expect((result.val as ShellExecOutput).exitCode).toBe(0); + }); + }); +}); diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts new file mode 100644 index 0000000000..461860a488 --- /dev/null +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts @@ -0,0 +1,284 @@ +import { Err, Ok, Result } from "ts-results"; +import { singleton } from "tsyringe"; +import { WebSocket } from "ws"; + +const MAX_OUTPUT_SIZE = 1024 * 1024; + +export type ShellExecInput = { + providerBaseUrl: string; + providerAddress: string; + dseq: string; + gseq: number; + oseq: number; + service: string; + command: string; + timeout: number; + jwtToken: string; +}; + +export type ShellExecOutput = { + stdout: string; + stderr: string; + exitCode: number; + truncated: boolean; +}; + +@singleton() +export class ShellExecService { + constructor() {} + + async execute(input: ShellExecInput): Promise> { + const url = buildShellUrl(input); + const auth = { type: "jwt" as const, token: input.jwtToken }; + + return new Promise(resolve => { + const timeoutId = setTimeout(() => { + ws.close(); + resolve(Err("Command timed out")); + }, input.timeout * 1000); + + let stdout = ""; + let stderr = ""; + let exitCode: number | undefined; + let truncated = false; + + const ws = new WebSocket(url, { + headers: { Authorization: `Bearer ${auth.token}` } + }); + + ws.on("open", () => { + const message: WebSocketOutgoingMessage = { + type: "websocket", + url, + auth, + providerAddress: input.providerAddress, + isBase64: true + }; + ws.send(JSON.stringify(message)); + }); + + ws.on("message", (data: Buffer) => { + try { + const message = JSON.parse(data.toString()) as ReceivedMessage; + + if (message.type === "pong") { + return; + } + + if (message.type === "websocket" && message.message) { + if (typeof message.message === "string") { + const parsed = parseShellMessage(message.message); + if (parsed) { + if (parsed.type === "exit_code") { + exitCode = parsed.exit_code; + clearTimeout(timeoutId); + ws.close(); + } else if (parsed.type === "data" && parsed.data) { + if (truncated) { + return; + } + const output = parsed.data; + if (output.length + stdout.length + stderr.length <= MAX_OUTPUT_SIZE) { + if (parsed.stream === "stdout") { + stdout += output; + } else { + stderr += output; + } + } else { + truncated = true; + } + } + } + } else if (Array.isArray(message.message)) { + const messageData = message.message; + if (messageData.length > 1) { + const firstByte = messageData[0]; + const payload = messageData.slice(1); + const textDecoder = new TextDecoder("utf-8"); + const output = textDecoder.decode(Buffer.from(payload)); + + if (firstByte === 0) { + exitCode = 0; + clearTimeout(timeoutId); + ws.close(); + } else if (firstByte === 1) { + if (truncated) { + return; + } + if (output.length + stdout.length + stderr.length <= MAX_OUTPUT_SIZE) { + stdout += output; + } else { + truncated = true; + } + } else if (firstByte === 2) { + if (truncated) { + return; + } + if (output.length + stderr.length + stdout.length <= MAX_OUTPUT_SIZE) { + stderr += output; + } else { + truncated = true; + } + } + } + + if (messageData.length === 1 && messageData[0] === 0) { + exitCode = 0; + clearTimeout(timeoutId); + ws.close(); + } + } + } + + if (message.closed || message.error) { + clearTimeout(timeoutId); + if (exitCode === undefined) { + if (message.error) { + resolve(Err(`Provider error: ${message.error}`)); + } else { + exitCode = 1; + } + } + ws.close(); + } + } catch { + // Ignore parse errors + } + }); + + ws.on("error", err => { + clearTimeout(timeoutId); + ws.close(); + resolve(Err(`WebSocket connection failed: ${err.message}`)); + }); + + ws.on("close", () => { + clearTimeout(timeoutId); + if (exitCode !== undefined) { + resolve( + Ok({ + stdout, + stderr, + exitCode, + truncated + }) + ); + } else { + resolve(Err("Connection closed without exit code")); + } + }); + }); + } +} + +/** + * Tokenizes a shell command into argv-style tokens the way a POSIX shell splits + * a command line, so each token maps to one `cmdN` query param consumed by the + * provider's lease-shell endpoint (which execs the tokens as argv, with no shell + * re-interpretation of its own). + * + * A naive `command.split(" ")` corrupts any argument that legitimately contains + * whitespace — quoted strings, paths, JSON blobs, PEM bodies — and is especially + * broken for the post-deploy secret-injection use case (e.g. + * `sh -c "echo SECRET=v > /run/secrets/.env"`), which must survive as the three + * tokens `["sh", "-c", "echo SECRET=v > /run/secrets/.env"]`. + * + * Rules: unquoted whitespace delimits tokens (space/tab/newline — matching the + * POSIX default `IFS`, so a carriage return is preserved, not a delimiter); single quotes + * preserve their contents literally; double quotes preserve contents but allow + * `\` to escape `"` and `\`; a backslash outside quotes escapes the next char. + * Quote characters are removed from the emitted tokens (shell semantics). + */ +export function tokenizeCommand(command: string): string[] { + const tokens: string[] = []; + let current = ""; + let hasToken = false; + let quote: '"' | "'" | null = null; + + for (let i = 0; i < command.length; i++) { + const char = command[i]; + + if (quote === "'") { + if (char === "'") quote = null; + else current += char; + continue; + } + + if (quote === '"') { + if (char === "\\" && (command[i + 1] === '"' || command[i + 1] === "\\")) { + current += command[++i]; + } else if (char === '"') { + quote = null; + } else { + current += char; + } + continue; + } + + if (char === "'" || char === '"') { + quote = char; + hasToken = true; + continue; + } + + if (char === "\\" && i + 1 < command.length) { + current += command[++i]; + hasToken = true; + continue; + } + + if (char === " " || char === "\t" || char === "\n") { + if (hasToken) { + tokens.push(current); + current = ""; + hasToken = false; + } + continue; + } + + current += char; + hasToken = true; + } + + if (hasToken) tokens.push(current); + return tokens; +} + +export function buildShellUrl(input: Pick): string { + const tokens = tokenizeCommand(input.command); + const cmdParts = tokens.length > 0 ? tokens.map((token, i) => `&cmd${i}=${encodeURIComponent(token)}`) : [`&cmd0=`]; + const baseUrl = input.providerBaseUrl.replace(/\/$/, ""); + return `${baseUrl}/lease/${encodeURIComponent(input.dseq)}/${input.gseq}/${input.oseq}/shell?stdin=0&tty=0&podIndex=0&service=${encodeURIComponent(input.service)}${cmdParts.join("")}`; +} + +export function parseShellMessage(message: string): { type: "data" | "exit_code"; data?: string; stream?: "stdout" | "stderr"; exit_code?: number } | null { + if (message.startsWith("{")) { + try { + const parsed = JSON.parse(message); + if (typeof parsed.exit_code === "number") { + return { type: "exit_code", exit_code: parsed.exit_code }; + } + if (typeof parsed.message === "string" && parsed.message.length > 0) { + return { type: "data", data: parsed.message, stream: "stdout" }; + } + } catch { + return null; + } + } + return null; +} + +type WebSocketOutgoingMessage = { + type: "websocket"; + url: string; + auth: { type: "jwt"; token: string }; + providerAddress: string; + isBase64: boolean; +}; + +type ReceivedMessage = { + type?: string; + message?: string | number[]; + closed?: boolean; + error?: string; +}; diff --git a/apps/api/src/routers/open-api-handlers.ts b/apps/api/src/routers/open-api-handlers.ts index e07926f3ad..dd55fa8fa1 100644 --- a/apps/api/src/routers/open-api-handlers.ts +++ b/apps/api/src/routers/open-api-handlers.ts @@ -33,6 +33,7 @@ import { import { deploymentSettingRouter } from "@src/deployment/routes/deployment-setting/deployment-setting.router"; import { deploymentsRouter } from "@src/deployment/routes/deployments/deployments.router"; import { leasesRouter } from "@src/deployment/routes/leases/leases.router"; +import { shellExecRouter } from "@src/deployment/routes/shell-exec/shell-exec.router"; import { gpuRouter } from "@src/gpu"; import { pricingRouter } from "@src/pricing"; import { proposalsRouter } from "@src/proposal"; @@ -77,6 +78,7 @@ export const openApiHonoHandlers: OpenApiHonoHandler[] = [ deploymentSettingRouter, deploymentsRouter, leasesRouter, + shellExecRouter, apiKeysRouter, bidsRouter, certificateRouter, diff --git a/apps/api/src/routes/deployment/index.ts b/apps/api/src/routes/deployment/index.ts index 57b5140b09..c55f70a1bf 100644 --- a/apps/api/src/routes/deployment/index.ts +++ b/apps/api/src/routes/deployment/index.ts @@ -2,5 +2,6 @@ import { bidsRouter } from "@src/bid/routes/bids/bids.router"; import { certificateRouter } from "@src/certificate/routes/certificate.router"; import { deploymentsRouter } from "@src/deployment/routes/deployments/deployments.router"; import { leasesRouter } from "@src/deployment/routes/leases/leases.router"; +import { shellExecRouter } from "@src/deployment/routes/shell-exec/shell-exec.router"; -export default [deploymentsRouter, bidsRouter, certificateRouter, leasesRouter]; +export default [deploymentsRouter, bidsRouter, certificateRouter, leasesRouter, shellExecRouter]; diff --git a/package-lock.json b/package-lock.json index cb9f77f9e6..48e59317f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -104,6 +104,7 @@ "ts-results": "^3.3.0", "tsyringe": "^4.10.0", "unleash-client": "^6.6.0", + "ws": "^8.18.2", "zod": "3.*" }, "devDependencies": { @@ -124,6 +125,7 @@ "@types/pg": "^8.11.6", "@types/semver": "^7.5.2", "@types/tar": "^6.1.13", + "@types/ws": "^8.5.4", "@typescript-eslint/eslint-plugin": "^7.12.0", "@vitest/coverage-v8": "^4.1.5", "drizzle-kit": "^0.22.7", From b960b54a129a3c27d09f5a69f6f0c869ca6ebadc Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Tue, 14 Jul 2026 10:48:58 +0100 Subject: [PATCH 2/4] fix(deployment): shell-exec via provider-proxy, argv command, stdin-secret injection Address review feedback on the synchronous shell-exec endpoint: - Route the shell WebSocket through provider-proxy (PROVIDER_PROXY_URL) instead of a direct provider connection, so self-signed provider certs are validated by the proxy. Rewrite the receive parser to the real proxy frame protocol: message.message.data with LeaseShellCode markers (100 stdout / 101 stderr / 102 result / 103 failure), error frames handled before base64-decode, dual JSON / little-endian-int32 exit-code, 4001/4003 auth-expiry handling, and keep-reading-after-1MB-truncation. - Switch the command contract to argv (command: string[]); remove tokenizeCommand. - Add stdin-secret injection: optional `stdin` streamed as a 104 frame (+ EOF) so secrets never appear in the (logged) provider URL. - Residuals: dseq numeric regex, OpenAPI 500 response, provider lookup before JWT mint, typed test factories. Co-Authored-By: Claude Opus 4.8 --- .../shell-exec/shell-exec.controller.spec.ts | 177 +- .../shell-exec/shell-exec.controller.ts | 19 +- .../http-schemas/shell-exec.schema.ts | 11 +- .../routes/shell-exec/shell-exec.router.ts | 3 + .../shell-exec/shell-exec.service.spec.ts | 1746 ++++------------- .../services/shell-exec/shell-exec.service.ts | 478 +++-- 6 files changed, 846 insertions(+), 1588 deletions(-) diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts index d32b3080c2..91e11076c9 100644 --- a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts @@ -12,12 +12,54 @@ import { ShellExecController } from "./shell-exec.controller"; import { createUser } from "@test/seeders/user.seeder"; +type DeploymentResponse = NonNullable>>; +type Lease = DeploymentResponse["leases"][number]; +type ProviderInfo = NonNullable>>; + +function createLease(overrides: Partial<{ gseq: number; oseq: number; provider: string; state: string }> = {}): Lease { + return { + id: { owner: "akash1owner", dseq: "1234", gseq: overrides.gseq ?? 1, oseq: overrides.oseq ?? 1, provider: overrides.provider ?? "akash1provider", bseq: 0 }, + state: overrides.state ?? "active", + price: { denom: "uakt", amount: "100" }, + created_at: "12345", + closed_on: "0", + status: null + } as unknown as Lease; +} + +function createDeployment(overrides: Partial<{ state: string; leases: Lease[] }> = {}): DeploymentResponse { + return { + deployment: { + id: { owner: "akash1owner", dseq: "1234" }, + state: overrides.state ?? "active", + hash: "abc123", + created_at: "12345" + }, + leases: overrides.leases ?? [createLease()], + escrow_account: { + id: { scope: "deployment", xid: "1234" }, + state: { + owner: "akash1owner", + state: "open", + transferred: [], + settled_at: "12345", + funds: [{ denom: "uakt", amount: "1000" }], + deposits: [] + } + } + } as unknown as DeploymentResponse; +} + +function createProviderInfo(overrides: Partial<{ hostUri: string }> = {}): ProviderInfo { + return { hostUri: overrides.hostUri ?? "https://provider.example.com" } as unknown as ProviderInfo; +} + describe(ShellExecController.name, () => { it("throws 404 when deployment not found", async () => { const { controller, deploymentReaderService } = setup(); - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(undefined as never); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(undefined as unknown as DeploymentResponse); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(404); expect(error.message).toBe("Deployment not found"); @@ -26,7 +68,7 @@ describe(ShellExecController.name, () => { it("throws 404 when no lease matches the provided gseq and oseq", async () => { const { controller } = setup(); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 99, oseq: 99, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 99, oseq: 99, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(404); expect(error.message).toBe("Lease not found"); @@ -35,7 +77,7 @@ describe(ShellExecController.name, () => { it("throws 500 when lease provider address is an empty string", async () => { const { controller } = setup({ provider: "" }); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(500); expect(error.message).toBe("Lease provider address not found"); @@ -44,7 +86,7 @@ describe(ShellExecController.name, () => { it("throws 400 when lease state is not active", async () => { const { controller } = setup({ state: "closed" }); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(400); expect(error.message).toBe("Lease is not active"); @@ -54,7 +96,7 @@ describe(ShellExecController.name, () => { const { controller, shellExecService } = setup(); shellExecService.execute.mockResolvedValue(Err("Command timed out")); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(502); expect(error.message).toBe("Command execution timed out"); @@ -63,7 +105,7 @@ describe(ShellExecController.name, () => { it("returns the shell exec result on successful execution", async () => { const { controller, shellExecService } = setup(); - const result = await controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 }); + const result = await controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 }); expect(result).toEqual({ stdout: "output", stderr: "", exitCode: 0, truncated: false }); expect(shellExecService.execute).toHaveBeenCalledWith({ @@ -73,56 +115,81 @@ describe(ShellExecController.name, () => { gseq: 1, oseq: 1, service: "web", - command: "ls", + command: ["ls"], timeout: 60, jwtToken: "test-token" }); }); + it("forwards stdin through to the shell exec service", async () => { + const { controller, shellExecService } = setup(); + + await controller.exec({ + dseq: "1234", + gseq: 1, + oseq: 1, + command: ["sh", "-c", "cat > /run/secrets/.env"], + service: "web", + timeout: 60, + stdin: "SECRET=value" + }); + + expect(shellExecService.execute).toHaveBeenCalledWith(expect.objectContaining({ stdin: "SECRET=value" })); + }); + it("throws 404 when provider info lookup returns null", async () => { const { controller, providerService } = setup(); - providerService.getProvider.mockResolvedValue(null as never); + providerService.getProvider.mockResolvedValue(null); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(404); expect(error.message).toBe("Provider not found"); }); + it("does not mint a provider JWT when the provider lookup fails (getProvider precedes toProviderAuth)", async () => { + const { controller, providerService } = setup(); + providerService.getProvider.mockResolvedValue(null); + + await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); + + expect(providerService.toProviderAuth).not.toHaveBeenCalled(); + }); + it("throws 404 when deployment has an empty leases array", async () => { - const { controller, deploymentReaderService, deployment } = setup(); - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue({ ...deployment, leases: [] } as never); + const { controller, deploymentReaderService } = setup(); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(createDeployment({ leases: [] })); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(404); expect(error.message).toBe("Lease not found"); }); it("finds the correct lease among multiple leases by gseq and oseq", async () => { - const { controller, deploymentReaderService, deployment, shellExecService } = setup(); + const { controller, deploymentReaderService, shellExecService } = setup(); - const multiLeaseDeployment = { - ...deployment, - leases: [ - { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 1, oseq: 1 }, state: "active" }, - { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 2, oseq: 1, provider: "akash1provider2" }, state: "active" }, - { ...deployment.leases[0], id: { ...deployment.leases[0].id, gseq: 1, oseq: 2, provider: "akash1provider3" }, state: "active" } - ] - }; - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(multiLeaseDeployment as never); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue( + createDeployment({ + leases: [ + createLease({ gseq: 1, oseq: 1 }), + createLease({ gseq: 2, oseq: 1, provider: "akash1provider2" }), + createLease({ gseq: 1, oseq: 2, provider: "akash1provider3" }) + ] + }) + ); - const result = await controller.exec({ dseq: "1234", gseq: 2, oseq: 1, command: "ls", service: "web", timeout: 60 }); + const result = await controller.exec({ dseq: "1234", gseq: 2, oseq: 1, command: ["ls"], service: "web", timeout: 60 }); expect(result).toEqual({ stdout: "output", stderr: "", exitCode: 0, truncated: false }); expect(shellExecService.execute).toHaveBeenCalledWith(expect.objectContaining({ providerAddress: "akash1provider2" })); }); it("throws 400 when deployment state is closed", async () => { - const { controller, deploymentReaderService, deployment } = setup(); - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue({ ...deployment, deployment: { ...deployment.deployment, state: "closed" } } as never); + const { controller, deploymentReaderService } = setup(); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(createDeployment({ state: "closed" })); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(400); expect(error.message).toBe("Deployment is not active"); @@ -132,7 +199,7 @@ describe(ShellExecController.name, () => { const { controller, shellExecService } = setup(); shellExecService.execute.mockResolvedValue(Err("WebSocket connection failed: ECONNREFUSED")); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(502); expect(error.message).toBe("Failed to connect to provider"); @@ -142,18 +209,28 @@ describe(ShellExecController.name, () => { const { controller, shellExecService } = setup(); shellExecService.execute.mockResolvedValue(Err("Provider error: internal error")); - const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: "ls", service: "web", timeout: 60 })); + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); expect(error.status).toBe(502); expect(error.message).toBe("Provider returned an error"); }); - async function captureError(fn: () => Promise): Promise { + it("throws 502 with auth-expired message when the provider JWT expires mid-run", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("Auth expired: provider closed connection (code 4001)")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Provider authentication expired"); + }); + + async function captureError(fn: () => Promise): Promise<{ status: number; message: string }> { try { await fn(); throw new Error("Expected function to throw"); } catch (error) { - return error; + return error as { status: number; message: string }; } } @@ -172,40 +249,14 @@ describe(ShellExecController.name, () => { const provider = overrides?.provider ?? "akash1provider"; const state = overrides?.state ?? "active"; - const deployment = { - deployment: { - id: { owner: "akash1owner", dseq: "1234" }, - state: "active", - hash: "abc123", - created_at: "12345" - }, - leases: [ - { - id: { owner: "akash1owner", dseq: "1234", gseq: 1, oseq: 1, provider, bseq: 0 }, - state, - price: { denom: "uakt", amount: "100" }, - created_at: "12345", - closed_on: "0", - status: null - } - ], - escrow_account: { - id: { scope: "deployment", xid: "1234" }, - state: { - owner: "akash1owner", - state: "open", - transferred: [], - settled_at: "12345", - funds: [{ denom: "uakt", amount: "1000" }], - deposits: [] - } - } - }; + const deployment = createDeployment({ leases: [createLease({ provider, state })] }); - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(deployment as never); - walletReaderService.getWalletByUserId.mockResolvedValue({ id: 1, address: "akash1wallet" } as never); + deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(deployment); + walletReaderService.getWalletByUserId.mockResolvedValue({ id: 1, address: "akash1wallet" } as Awaited< + ReturnType + >); providerService.toProviderAuth.mockResolvedValue({ type: "jwt" as const, token: "test-token" }); - providerService.getProvider.mockResolvedValue({ hostUri: "https://provider.example.com" } as never); + providerService.getProvider.mockResolvedValue(createProviderInfo()); shellExecService.execute.mockResolvedValue(new Ok({ stdout: "output", stderr: "", exitCode: 0, truncated: false })); return { controller, deploymentReaderService, providerService, shellExecService, authService, walletReaderService, user, deployment }; diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts index 68b8adaa9d..b723778e54 100644 --- a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts @@ -35,14 +35,14 @@ export class ShellExecController { const providerAddress = lease.id.provider; - const wallet = await this.walletReaderService.getWalletByUserId(userId); - - const auth = await this.providerService.toProviderAuth({ walletId: wallet.id, provider: providerAddress }, ["shell"]); - const providerInfo = await this.providerService.getProvider(providerAddress); assert(providerInfo, 404, "Provider not found"); + const wallet = await this.walletReaderService.getWalletByUserId(userId); + + const auth = await this.providerService.toProviderAuth({ walletId: wallet.id, provider: providerAddress }, ["shell"]); + const result = await this.shellExecService.execute({ providerBaseUrl: providerInfo.hostUri, providerAddress: providerAddress, @@ -51,6 +51,7 @@ export class ShellExecController { oseq: input.oseq, service: input.service, command: input.command, + stdin: input.stdin, timeout: input.timeout, jwtToken: auth.token }); @@ -60,9 +61,13 @@ export class ShellExecController { ? "Command execution timed out" : result.val.startsWith("WebSocket connection failed") ? "Failed to connect to provider" - : result.val.startsWith("Provider error") - ? "Provider returned an error" - : "Shell execution failed"; + : result.val.startsWith("Auth expired") + ? "Provider authentication expired" + : result.val.startsWith("Invalid provider host") + ? "Invalid provider host" + : result.val.startsWith("Provider error") + ? "Provider returned an error" + : "Shell execution failed"; assert(false, 502, message); } diff --git a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts index e1246af22d..c58a612560 100644 --- a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts +++ b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts @@ -1,15 +1,20 @@ import { z } from "zod"; export const ShellExecParamsSchema = z.object({ - dseq: z.string(), + dseq: z.string().regex(/^\d+$/), gseq: z.coerce.number().int().nonnegative(), oseq: z.coerce.number().int().nonnegative() }); export const ShellExecRequestSchema = z.object({ - command: z.string().min(1).max(4096), + command: z.array(z.string().min(1)).min(1).max(64), service: z.string().min(1).max(253), - timeout: z.number().int().min(1).max(120).default(60) + timeout: z.number().int().min(1).max(120).default(60), + stdin: z.string().max(1_048_576).optional().openapi({ + description: + 'Optional raw UTF-8 data streamed to the command\'s standard input (max 1 MiB). Put secrets HERE (env files, tokens, passwords) — never in `command`, whose tokens are placed in the provider-proxy request URL, which is logged. Example command: ["sh","-c","cat > /run/secrets/.env"].', + example: "SECRET=value" + }) }); export const ShellExecResponseSchema = z.object({ diff --git a/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts index 8319a3901a..f7cb269edc 100644 --- a/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts +++ b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts @@ -45,6 +45,9 @@ const shellExecRoute = createRoute({ 404: { description: "Deployment or lease not found" }, + 500: { + description: "Internal server error (e.g., lease provider address missing)" + }, 502: { description: "Provider proxy error" } diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts index a92806ea45..70a23542f3 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts @@ -1,193 +1,106 @@ import { faker } from "@faker-js/faker"; import { describe, expect, it, vi } from "vitest"; +import { mock } from "vitest-mock-extended"; import { WebSocket } from "ws"; -import { buildShellUrl, parseShellMessage, type ShellExecOutput, ShellExecService } from "./shell-exec.service"; +import type { LoggerService } from "@src/core"; +import type { DeploymentConfig } from "@src/deployment/config/config.provider"; +import { + buildShellUrl, + isStrictBase64, + isValidProviderHost, + parseExitCode, + type ShellExecInput, + type ShellExecOutput, + ShellExecService, + toProxyWebSocketUrl +} from "./shell-exec.service"; vi.mock("ws", () => ({ WebSocket: vi.fn() })); -describe(ShellExecService.name, () => { - describe("buildShellUrl", () => { - it("builds correct URL with single word command", () => { - const input = createShellExecInput({ command: "ls" }); - - const url = buildShellUrl(input); - - expect(url).toContain("/lease/test-dseq/1/1/shell"); - expect(url).toContain("stdin=0"); - expect(url).toContain("tty=0"); - expect(url).toContain("podIndex=0"); - expect(url).toContain("service=test-service"); - expect(url).toContain("cmd0=ls"); - }); - - it("builds correct URL with multi-word command", () => { - const input = createShellExecInput({ command: "echo hello world" }); +const PROXY_URL = "https://proxy.example.com"; - const url = buildShellUrl(input); - - expect(url).toContain("/lease/test-dseq/1/1/shell"); - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); - expect(url).toContain("cmd2=world"); +describe(ShellExecService.name, () => { + describe("toProxyWebSocketUrl", () => { + it("maps https to wss", () => { + expect(toProxyWebSocketUrl("https://proxy.example.com")).toBe("wss://proxy.example.com"); }); - it("builds correct URL with command containing path", () => { - const input = createShellExecInput({ command: "cat /run/secrets/db_password" }); - - const url = buildShellUrl(input); - - expect(url).toContain("/lease/test-dseq/1/1/shell"); - expect(url).toContain("cmd0=cat"); - expect(url).toContain("cmd1=%2Frun%2Fsecrets%2Fdb_password"); + it("maps http to ws", () => { + expect(toProxyWebSocketUrl("http://localhost:3000")).toBe("ws://localhost:3000"); }); + }); - it("removes trailing slash from provider base URL", () => { - const input = createShellExecInput({ providerBaseUrl: "https://provider.example.com/" }); - - const url = buildShellUrl(input); - - expect(url).toMatch(/^https:\/\/provider\.example\.com\/lease/); + describe("isValidProviderHost", () => { + it("accepts an https host with a domain name", () => { + expect(isValidProviderHost("https://provider.example.com:8443")).toBe(true); }); - it("handles special characters in command", () => { - const input = createShellExecInput({ command: "echo hello&world" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello%26world"); + it("rejects a non-https (http) host", () => { + expect(isValidProviderHost("http://provider.example.com")).toBe(false); }); - it("encodes service name with spaces", () => { - const input = createShellExecInput({ service: "my service" }); - - const url = buildShellUrl(input); - - expect(url).toContain("service=my%20service"); + it("rejects an IPv4 host", () => { + expect(isValidProviderHost("https://203.0.113.10:8443")).toBe(false); }); - it("builds URL with correct gseq and oseq", () => { - const input = createShellExecInput({ gseq: 3, oseq: 5 }); - - const url = buildShellUrl(input); - - expect(url).toContain("/lease/test-dseq/3/5/shell"); + it("rejects an IPv6 host", () => { + expect(isValidProviderHost("https://[2001:db8::1]:8443")).toBe(false); }); - it("handles empty command string", () => { - const input = createShellExecInput({ command: "" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0="); - expect(url).not.toContain("cmd1="); + it("rejects a .local host", () => { + expect(isValidProviderHost("https://provider.local:8443")).toBe(false); }); - it("handles whitespace-only command string", () => { - const input = createShellExecInput({ command: " " }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0="); - expect(url).not.toContain("cmd1="); + it("rejects a malformed URL", () => { + expect(isValidProviderHost("not a url")).toBe(false); }); + }); - it("handles command with consecutive spaces", () => { - const input = createShellExecInput({ command: "echo hello" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); + describe("isStrictBase64", () => { + it("accepts a valid base64 string", () => { + expect(isStrictBase64(Buffer.from("hello").toString("base64"))).toBe(true); }); - it("whitespace-only command should not produce multiple cmd params", () => { - const input = createShellExecInput({ command: " " }); - - const url = buildShellUrl(input); - - expect(url).not.toContain("cmd1="); - expect(url).not.toContain("cmd2="); - expect(url).not.toContain("cmd3="); + it("rejects prose with spaces", () => { + expect(isStrictBase64("Received error from provider websocket")).toBe(false); }); - it("treats newline as a token delimiter", () => { - const input = createShellExecInput({ command: "echo\nhello" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); - expect(url).not.toContain("cmd2="); + it("rejects a string whose length is not a multiple of four", () => { + expect(isStrictBase64("abc")).toBe(false); }); - it("command with leading spaces should filter them out", () => { - const input = createShellExecInput({ command: " echo hello" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); - expect(url).not.toContain("cmd2="); + it("rejects an empty string", () => { + expect(isStrictBase64("")).toBe(false); }); + }); - it("treats tab as a token delimiter", () => { - const input = createShellExecInput({ command: "echo\thello" }); - - const url = buildShellUrl(input); + describe("buildShellUrl (argv mapping)", () => { + it("maps a single-token argv to cmd0", () => { + const url = buildShellUrl(createShellExecInput({ command: ["ls"] })); - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); - expect(url).not.toContain("cmd2="); + expect(url).toContain("/lease/1234/1/1/shell"); + expect(url).toContain("stdin=0"); + expect(url).toContain("tty=0"); + expect(url).toContain("podIndex=0"); + expect(url).toContain("service=test-service"); + expect(url).toContain("cmd0=ls"); + expect(url).not.toContain("cmd1="); }); - it("collapses mixed whitespace (spaces and tabs) between tokens", () => { - const input = createShellExecInput({ command: "echo \t hello" }); - - const url = buildShellUrl(input); + it("maps a multi-token argv to cmd0..cmdN", () => { + const url = buildShellUrl(createShellExecInput({ command: ["echo", "hello", "world"] })); expect(url).toContain("cmd0=echo"); expect(url).toContain("cmd1=hello"); - expect(url).not.toContain("cmd2="); - }); - - it("splits on multiple consecutive tabs", () => { - const input = createShellExecInput({ command: "cmd\t\targ" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=cmd"); - expect(url).toContain("cmd1=arg"); - expect(url).not.toContain("cmd2="); - }); - - it("keeps a double-quoted argument with spaces as a single token", () => { - const input = createShellExecInput({ command: 'echo "hello world"' }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello%20world"); - expect(url).not.toContain("cmd2="); - }); - - it("keeps a single-quoted argument with spaces as a single token", () => { - const input = createShellExecInput({ command: "echo 'a b c'" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=a%20b%20c"); - expect(url).not.toContain("cmd2="); + expect(url).toContain("cmd2=world"); + expect(url).not.toContain("cmd3="); }); - it("preserves the post-deploy secret-injection command as argv (sh -c ...)", () => { - const input = createShellExecInput({ command: 'sh -c "echo SECRET=v > /run/secrets/.env"' }); - - const url = buildShellUrl(input); + it("URL-encodes each argv token independently, preserving whitespace inside a token", () => { + const url = buildShellUrl(createShellExecInput({ command: ["sh", "-c", "echo SECRET=v > /run/secrets/.env"] })); expect(url).toContain("cmd0=sh"); expect(url).toContain("cmd1=-c"); @@ -195,1317 +108,524 @@ describe(ShellExecService.name, () => { expect(url).not.toContain("cmd3="); }); - it("treats a backslash-escaped space as part of the token", () => { - const input = createShellExecInput({ command: "echo a\\ b" }); - - const url = buildShellUrl(input); + it("encodes reserved characters in a token", () => { + const url = buildShellUrl(createShellExecInput({ command: ["echo", "a&b?c#d"] })); expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=a%20b"); - expect(url).not.toContain("cmd2="); + expect(url).toContain("cmd1=a%26b%3Fc%23d"); }); - it("handles service name with hash fragment character", () => { - const input = createShellExecInput({ service: "svc#1" }); - - const url = buildShellUrl(input); + it("encodes the service name", () => { + const url = buildShellUrl(createShellExecInput({ service: "my service" })); - expect(url).toContain("service=svc%231"); + expect(url).toContain("service=my%20service"); }); - it("handles service name with question mark", () => { - const input = createShellExecInput({ service: "my?service" }); - - const url = buildShellUrl(input); + it("uses the provided gseq and oseq in the path", () => { + const url = buildShellUrl(createShellExecInput({ gseq: 3, oseq: 5 })); - expect(url).toContain("service=my%3Fservice"); + expect(url).toContain("/lease/1234/3/5/shell"); }); - it("handles provider base URL without trailing slash", () => { - const input = createShellExecInput({ providerBaseUrl: "https://provider.example.com" }); - - const url = buildShellUrl(input); + it("removes a trailing slash from the provider base URL", () => { + const url = buildShellUrl(createShellExecInput({ providerBaseUrl: "https://provider.example.com/" })); expect(url).toMatch(/^https:\/\/provider\.example\.com\/lease/); }); - it("command with carriage return should be URL-encoded", () => { - const input = createShellExecInput({ command: "echo\rhello" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo%0Dhello"); - expect(url).not.toContain("cmd1="); - }); - - it("handles Unicode characters in command", () => { - const input = createShellExecInput({ command: "echo 日本語" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=%E6%97%A5%E6%9C%AC%E8%AA%9E"); - }); - - it("handles gseq of 0 in URL path", () => { - const input = createShellExecInput({ gseq: 0 }); - - const url = buildShellUrl(input); + it("URL-encodes the dseq path segment", () => { + const url = buildShellUrl(createShellExecInput({ dseq: "foo/bar" })); - expect(url).toContain("/lease/test-dseq/0/1/shell"); + expect(url).toContain("/lease/foo%2Fbar/1/1/shell"); }); + }); - it("handles oseq of 0 in URL path", () => { - const input = createShellExecInput({ oseq: 0 }); - - const url = buildShellUrl(input); + describe("buildShellUrl (stdin flag)", () => { + it("emits stdin=0 when no stdin is provided", () => { + const url = buildShellUrl(createShellExecInput()); - expect(url).toContain("/lease/test-dseq/1/0/shell"); + expect(url).toContain("stdin=0"); + expect(url).not.toContain("stdin=1"); }); - it("builds URL with empty dseq (all slashes preserved)", () => { - const input = createShellExecInput({ dseq: "" }); - - const url = buildShellUrl(input); + it("emits stdin=1 when stdin is provided", () => { + const url = buildShellUrl(createShellExecInput({ stdin: "SECRET=topsecret" })); - expect(url).toContain("/lease//1/1/shell"); + expect(url).toContain("stdin=1"); + expect(url).not.toContain("stdin=0"); }); - it("builds URL with empty service name", () => { - const input = createShellExecInput({ service: "" }); + it("emits stdin=0 for an empty stdin string", () => { + const url = buildShellUrl(createShellExecInput({ stdin: "" })); - const url = buildShellUrl(input); - - expect(url).toContain("service="); + expect(url).toContain("stdin=0"); + expect(url).not.toContain("stdin=1"); }); - it("builds URL with empty providerBaseUrl (should not produce double slash)", () => { - const input = createShellExecInput({ providerBaseUrl: "" }); + it("never places the stdin payload in the URL", () => { + const secret = "SUPER_SECRET_VALUE_12345"; + const url = buildShellUrl(createShellExecInput({ command: ["sh", "-c", "cat > /run/secrets/.env"], stdin: secret })); - const url = buildShellUrl(input); - - expect(url).toMatch(/^\/lease/); + expect(url).not.toContain(secret); + expect(url).not.toContain(encodeURIComponent(secret)); }); }); - describe("binary message stream marker edge cases", () => { - it("binary message with stream marker 3 (unexpected) should not corrupt state", () => { - const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); - - expect(result).toBeNull(); + describe("parseExitCode", () => { + it("parses a JSON exit_code body", () => { + expect(parseExitCode(Buffer.from('{"exit_code":42}'))).toBe(42); }); - it("binary message with stream marker 255 (unexpected) should not corrupt state", () => { - const result = parseShellMessage("[255, 72]"); - - expect(result).toBeNull(); - }); - }); - - describe("parseShellMessage", () => { - it("should return exit_code parsed from JSON message", () => { - const result = parseShellMessage('{"exit_code": 42}'); - - expect(result).toEqual({ type: "exit_code", exit_code: 42 }); + it("parses a JSON exit_code of 0", () => { + expect(parseExitCode(Buffer.from('{"exit_code":0}'))).toBe(0); }); - it("should return data with stream from JSON message containing message field", () => { - const result = parseShellMessage('{"message": "hello"}'); - - expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); + it("maps a JSON null exit_code to 0", () => { + expect(parseExitCode(Buffer.from('{"exit_code":null}'))).toBe(0); }); - it("should return null for non-JSON string message", () => { - const result = parseShellMessage("plain text"); - - expect(result).toBeNull(); + it("parses a 4-byte little-endian int32 payload", () => { + expect(parseExitCode(Buffer.from([42, 0, 0, 0]))).toBe(42); }); - }); - describe("parseShellMessage edge cases", () => { - it("returns null for empty JSON object", () => { - const result = parseShellMessage("{}"); - - expect(result).toBeNull(); + it("parses a 4-byte LE int32 whose first byte is 0x7B ('{')", () => { + expect(parseExitCode(Buffer.from([123, 0, 0, 0]))).toBe(123); }); - it("returns null for JSON with null values", () => { - const result = parseShellMessage('{"exit_code": null, "message": null}'); - - expect(result).toBeNull(); + it("returns 0 for an empty payload", () => { + expect(parseExitCode(Buffer.from([]))).toBe(0); }); + }); - it("prefers exit_code over message when both present", () => { - const result = parseShellMessage('{"exit_code": 0, "message": "ignored"}'); - - expect(result).toEqual({ type: "exit_code", exit_code: 0 }); - }); + describe("execute - provider host pre-check", () => { + it("returns an Err without opening a socket when the host is not https", async () => { + const { service } = createService(); - it("returns null for JSON with undefined values", () => { - const result = parseShellMessage('{"exit_code": undefined}'); + const result = await service.execute(createShellExecInput({ providerBaseUrl: "http://provider.example.com" })); - expect(result).toBeNull(); + expect(result.ok).toBe(false); + expect(result.val).toContain("Invalid provider host"); + expect(vi.mocked(WebSocket)).not.toHaveBeenCalled(); }); - it("returns null for malformed JSON", () => { - const result = parseShellMessage('{"exit_code": }'); - - expect(result).toBeNull(); - }); + it("returns an Err when the host is an IP address", async () => { + const { service } = createService(); - it("returns null when exit_code is string instead of number", () => { - const result = parseShellMessage('{"exit_code": "42"}'); + const result = await service.execute(createShellExecInput({ providerBaseUrl: "https://203.0.113.10:8443" })); - expect(result).toBeNull(); + expect(result.ok).toBe(false); + expect(result.val).toContain("Invalid provider host"); }); + }); - it("returns null when exit_code is boolean true", () => { - const result = parseShellMessage('{"exit_code": true}'); - - expect(result).toBeNull(); - }); + describe("execute - proxy routing (D1)", () => { + it("connects to the PROVIDER_PROXY_URL as wss and sends the envelope with the provider url and no data", async () => { + const { service, mockWs, getConstructorArgs } = createService(); + const input = createShellExecInput(); - it("returns null when exit_code is boolean false", () => { - const result = parseShellMessage('{"exit_code": false}'); + const promise = service.execute(input); + mockWs._trigger("open"); - expect(result).toBeNull(); + const [socketUrl, socketOptions] = getConstructorArgs(); + expect(socketUrl).toBe("wss://proxy.example.com"); + // No connection-level Authorization header: auth travels inside the envelope. + expect(socketOptions).toBeUndefined(); + + expect(mockWs.send).toHaveBeenCalledTimes(1); + const envelope = JSON.parse(mockWs.send.mock.calls[0][0]); + expect(envelope.type).toBe("websocket"); + expect(envelope.url).toBe(buildShellUrl(input)); + expect(envelope.url.startsWith("https://provider.example.com/lease/")).toBe(true); + expect(envelope.auth).toEqual({ type: "jwt", token: input.jwtToken }); + expect(envelope.providerAddress).toBe(input.providerAddress); + expect(envelope.isBase64).toBe(true); + expect("data" in envelope).toBe(false); + + // finish so the promise settles + mockWs._trigger("message", exitFrameJson(0)); + await promise; }); + }); - it("returns null when message is number instead of string", () => { - const result = parseShellMessage('{"message": 123}'); + describe("execute - stdin injection (Task 2b)", () => { + const STDIN_MARKER = 104; - expect(result).toBeNull(); - }); + it("sends a 104 stdin data frame plus a 104 EOF frame, keeping the secret out of the URL", async () => { + const { service, mockWs, getConstructorArgs } = createService(); + const secret = "SECRET=topsecret\nAPI_KEY=abc123"; + const input = createShellExecInput({ + command: ["sh", "-c", "cat > /run/secrets/.env && chmod 600 /run/secrets/.env"], + stdin: secret + }); - it("returns null for array input masquerading as JSON object", () => { - const result = parseShellMessage("[]"); + const promise = service.execute(input); + mockWs._trigger("open"); - expect(result).toBeNull(); + // Three frames: connect envelope, stdin data, stdin EOF. + expect(mockWs.send).toHaveBeenCalledTimes(3); + + const connect = JSON.parse(mockWs.send.mock.calls[0][0]); + const dataFrameMsg = JSON.parse(mockWs.send.mock.calls[1][0]); + const eofFrameMsg = JSON.parse(mockWs.send.mock.calls[2][0]); + + // Connect frame opens stdin (stdin=1) and carries no data. + expect(connect.url).toContain("stdin=1"); + expect("data" in connect).toBe(false); + + // The secret must not appear in the socket URL or the connect-frame url. + const [socketUrl] = getConstructorArgs(); + expect(socketUrl).toBe("wss://proxy.example.com"); + expect(String(socketUrl)).not.toContain(secret); + expect(connect.url).not.toContain(secret); + expect(connect.url).not.toContain(encodeURIComponent(secret)); + + // Data frame: full envelope + base64 of [104, ...secretBytes]. + expect(dataFrameMsg.type).toBe("websocket"); + expect(dataFrameMsg.url).toBe(connect.url); + expect(dataFrameMsg.auth).toEqual({ type: "jwt", token: input.jwtToken }); + expect(dataFrameMsg.providerAddress).toBe(input.providerAddress); + expect(dataFrameMsg.isBase64).toBe(true); + const dataBytes = Buffer.from(dataFrameMsg.data, "base64"); + expect(dataBytes[0]).toBe(STDIN_MARKER); + expect(dataBytes.subarray(1).toString("utf-8")).toBe(secret); + expect([...dataBytes]).toEqual([STDIN_MARKER, ...Buffer.from(secret, "utf-8")]); + + // The base64 payload itself must not be the plaintext secret (sanity). + expect(dataFrameMsg.data).not.toContain(secret); + + // EOF frame: full envelope + base64 of a bare [104] marker. + expect(eofFrameMsg.url).toBe(connect.url); + const eofBytes = Buffer.from(eofFrameMsg.data, "base64"); + expect([...eofBytes]).toEqual([STDIN_MARKER]); + + // A normal exit frame still yields the correct exit code. + mockWs._trigger("message", exitFrameJson(0)); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(0); }); - it("returns null for string input that looks like JSON but isn't", () => { - const result = parseShellMessage('"not an object"'); + it("does not send any 104 frame when stdin is omitted", async () => { + const { service, mockWs } = createService(); + const input = createShellExecInput(); - expect(result).toBeNull(); - }); + const promise = service.execute(input); + mockWs._trigger("open"); - it("returns null for number input", () => { - const result = parseShellMessage("123"); + // Only the connect envelope is sent. + expect(mockWs.send).toHaveBeenCalledTimes(1); - expect(result).toBeNull(); - }); + const connect = JSON.parse(mockWs.send.mock.calls[0][0]); + expect(connect.url).toContain("stdin=0"); + expect("data" in connect).toBe(false); - it("returns null for boolean JSON", () => { - const result = parseShellMessage("true"); + // No sent frame decodes to a 104 stdin marker. + const has104Frame = mockWs.send.mock.calls.some(([raw]) => { + const msg = JSON.parse(raw); + if (typeof msg.data !== "string") return false; + return Buffer.from(msg.data, "base64")[0] === STDIN_MARKER; + }); + expect(has104Frame).toBe(false); - expect(result).toBeNull(); + mockWs._trigger("message", exitFrameJson(0)); + const result = await promise; + expect(result.ok).toBe(true); }); }); - describe("binary message parsing edge cases", () => { - it("ignores empty binary array message", () => { - const result = parseShellMessage("[]"); - - expect(result).toBeNull(); - }); - - it("ignores binary message with only stream marker and no payload", () => { - const result = parseShellMessage("[1]"); - - expect(result).toBeNull(); - }); + describe("execute - receive / marker handling", () => { + it("routes marker 100 to stdout with the marker byte stripped", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("ignores binary message with invalid stream marker byte", () => { - const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(100, "hello")); + mockWs._trigger("message", exitFrameJson(0)); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("hello"); + expect((result.val as ShellExecOutput).stderr).toBe(""); + expect((result.val as ShellExecOutput).exitCode).toBe(0); }); - it("handles exit_code 0 as valid number", () => { - const result = parseShellMessage('{"exit_code": 0}'); - - expect(result).toEqual({ type: "exit_code", exit_code: 0 }); - }); + it("routes marker 101 to stderr with the marker byte stripped", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("handles negative exit_code", () => { - const result = parseShellMessage('{"exit_code": -1}'); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(101, "boom")); + mockWs._trigger("message", exitFrameJson(1)); - expect(result).toEqual({ type: "exit_code", exit_code: -1 }); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stderr).toBe("boom"); + expect((result.val as ShellExecOutput).stdout).toBe(""); + expect((result.val as ShellExecOutput).exitCode).toBe(1); }); - it("returns null for exit_code as NaN", () => { - const result = parseShellMessage('{"exit_code": NaN}'); - - expect(result).toBeNull(); - }); + it("reads the exit code from a 102 JSON result frame", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("returns null for exit_code as Infinity", () => { - const result = parseShellMessage('{"exit_code": Infinity}'); + mockWs._trigger("open"); + mockWs._trigger("message", exitFrameJson(7)); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(7); }); - it("returns null for exit_code as -Infinity", () => { - const result = parseShellMessage('{"exit_code": -Infinity}'); - - expect(result).toBeNull(); - }); + it("reads the exit code from a 102 4-byte LE int32 result frame", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("returns null for message as empty string", () => { - const result = parseShellMessage('{"message": ""}'); + mockWs._trigger("open"); + mockWs._trigger("message", bytesFrame([102, 9, 0, 0, 0])); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(9); }); - it("handles very large exit_code number", () => { - const result = parseShellMessage('{"exit_code": 9999999999}'); - - expect(result).toEqual({ type: "exit_code", exit_code: 9999999999 }); - }); + it("treats a 103 failure frame as a provider error (mapped 502), not output", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("should fall back to message when exit_code is non-numeric string", () => { - const result = parseShellMessage('{"exit_code": "invalid", "message": "hello"}'); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(103, "container terminated")); - expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("Provider error"); + expect(result.val).toContain("container terminated"); }); - it("should fall back to message when exit_code is object", () => { - const result = parseShellMessage('{"exit_code": {}, "message": "hello"}'); - - expect(result).toEqual({ type: "data", data: "hello", stream: "stdout" }); - }); + it("decodes a base64-string data payload", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("returns null for JSON starting with whitespace before brace", () => { - const result = parseShellMessage(' {"exit_code": 0}'); + const base64 = Buffer.from([100, ...Buffer.from("hi", "utf-8")]).toString("base64"); + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: { data: base64 } }))); + mockWs._trigger("message", exitFrameJson(0)); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("hi"); }); }); - describe("binary message edge cases", () => { - it("should handle exit_code 0 in JSON message", () => { - // JSON messages with exit_code are handled by parseShellMessage - const result = parseShellMessage('{"exit_code": 0}'); - - expect(result).toEqual({ type: "exit_code", exit_code: 0 }); - }); + describe("execute - error / robustness handling", () => { + it("resolves an error-key frame as a mapped provider error without decoding it", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - it("should return null for binary array with unexpected stream marker byte", () => { - // Stream marker 3 is not valid (only 1=stdout, 2=stderr are valid) - const result = parseShellMessage("[3, 72, 101, 108, 108, 111]"); + mockWs._trigger("open"); + mockWs._trigger( + "message", + Buffer.from( + JSON.stringify({ + type: "websocket", + message: "Message doesn't match expected schema", + error: "Invalid message format", + errors: [{ path: ["url"], message: "URL must use https protocol" }] + }) + ) + ); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("Provider error: Invalid message format"); + expect(result.val).toContain("url: URL must use https protocol"); }); - it("should return null for empty binary array", () => { - const result = parseShellMessage("[]"); + it("drops a non-base64 string payload instead of leaking it as output", async () => { + const { service, mockWs, logger } = createService(); + const promise = service.execute(createShellExecInput()); + + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: { data: "this is not base64 !!!" } }))); + mockWs._trigger("message", exitFrameJson(0)); - expect(result).toBeNull(); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe(""); + expect((result.val as ShellExecOutput).stderr).toBe(""); + expect(logger.warn).toHaveBeenCalled(); }); - }); - describe("buildShellUrl edge cases", () => { - it("treats an unquoted backslash as a shell escape (removes it)", () => { - const input = createShellExecInput({ command: "echo \\hello\\world" }); + it("ignores pong keepalive frames without corrupting output", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(100, "Hello ")); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "pong" }))); + mockWs._trigger("message", dataFrame(100, "World")); + mockWs._trigger("message", exitFrameJson(0)); - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=helloworld"); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).stdout).toBe("Hello World"); }); - it("preserves literal backslashes inside single quotes", () => { - const input = createShellExecInput({ command: "echo '\\hello\\world'" }); + it("returns an Err with the connection message on a socket error", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("error", new Error("ECONNREFUSED")); + mockWs._trigger("close"); - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=%5Chello%5Cworld"); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("WebSocket connection failed: ECONNREFUSED"); }); - it("URL-encodes dseq containing forward slash to prevent path traversal", () => { - const input = createShellExecInput({ dseq: "foo/bar/baz" }); + it("returns an Err when the socket closes before an exit code arrives", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("open"); + mockWs._trigger("close"); - expect(url).toContain("/lease/foo%2Fbar%2Fbaz/"); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Connection closed without exit code"); }); + }); - it("handles negative gseq by including it in URL path (no validation)", () => { - const input = createShellExecInput({ gseq: -1 }); + describe("execute - truncation (M5)", () => { + it("sets truncated but still reports the correct exit code when output exceeds 1 MB", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(100, "A".repeat(1024 * 1024 + 1))); + // keeps reading past the cap so the exit frame is still processed + mockWs._trigger("message", exitFrameJson(3)); - expect(url).toContain("/lease/test-dseq/-1/1/shell"); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).truncated).toBe(true); + expect((result.val as ShellExecOutput).exitCode).toBe(3); }); - it("handles negative oseq by including it in URL path (no validation)", () => { - const input = createShellExecInput({ oseq: -5 }); + it("does not truncate output that is exactly 1 MB", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + const data = "A".repeat(1024 * 1024); + mockWs._trigger("open"); + mockWs._trigger("message", dataFrame(100, data)); + mockWs._trigger("message", exitFrameJson(0)); - expect(url).toContain("/lease/test-dseq/1/-5/shell"); + const result = await promise; + expect(result.ok).toBe(true); + expect((result.val as ShellExecOutput).truncated).toBe(false); + expect((result.val as ShellExecOutput).stdout.length).toBe(1024 * 1024); }); + }); - it("command consisting only of tabs is trimmed to empty (tabs are whitespace)", () => { - const input = createShellExecInput({ command: "\t\t" }); + describe("execute - auth expiry (M4)", () => { + it("maps a 4001 close frame to an auth-expired error, not a generic provider error", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("open"); + mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: "", closed: true, code: 4001, reason: "token expired" }))); - expect(url).toContain("cmd0="); - expect(url).not.toContain("cmd1="); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("Auth expired"); + expect(result.val).not.toContain("Provider error"); }); - it("command with only newlines and tabs is trimmed to empty like tabs (all Unicode whitespace trimmed by JS trim)", () => { - const input = createShellExecInput({ command: "\n\t\n" }); + it("maps a 4003 ws close event to an auth-expired error", async () => { + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput()); - const url = buildShellUrl(input); + mockWs._trigger("open"); + mockWs._trigger("close", 4003, Buffer.from("unauthorized")); - expect(url).toContain("cmd0="); - expect(url).not.toContain("cmd1="); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toContain("Auth expired"); }); }); - describe("binary single-byte exit code edge cases", () => { - it("binary message [0] as single byte exit code should set exitCode to 0", () => { - const result = parseShellMessage("[0]"); + describe("execute - timeout", () => { + it("resolves with a timeout error when the command runs past the timeout", async () => { + vi.useFakeTimers(); + const { service, mockWs } = createService(); + const promise = service.execute(createShellExecInput({ timeout: 5 })); - expect(result).toBeNull(); - }); + mockWs._trigger("open"); + vi.advanceTimersByTime(5000); - it("binary message with stream marker 1 and no payload should be ignored", () => { - const result = parseShellMessage("[1]"); + const result = await promise; + expect(result.ok).toBe(false); + expect(result.val).toBe("Command timed out"); + expect(mockWs.close).toHaveBeenCalled(); - expect(result).toBeNull(); + vi.useRealTimers(); }); + }); - it("binary message [0, 72] should handle exit code 0 via firstByte check", () => { - const result = parseShellMessage("[0, 72]"); - - expect(result).toBeNull(); - }); - - it("binary message [0, 72, 101, 108, 108, 111] with stream marker 0 AND payload bytes should be ignored (exit code 0 followed by data is not valid)", () => { - const result = parseShellMessage("[0, 72, 101, 108, 108, 111]"); - - expect(result).toBeNull(); - }); - - it("NaN exit_code should be rejected (typeof NaN === 'number' in JS)", () => { - const result = parseShellMessage('{"exit_code": NaN}'); - - expect(result).toBeNull(); - }); - }); - - describe("parseShellMessage edge cases", () => { - it("returns null for JSON string null literal", () => { - const result = parseShellMessage("null"); + function createMockWebSocket() { + const handlers: Record void>> = {}; + return { + on: vi.fn((event: string, handler: (...args: unknown[]) => void) => { + (handlers[event] ??= []).push(handler); + }), + send: vi.fn(), + close: vi.fn(), + _trigger(event: string, ...args: unknown[]) { + (handlers[event] || []).forEach(h => h(...args)); + } + }; + } - expect(result).toBeNull(); - }); + function createService() { + const mockWs = createMockWebSocket(); + let constructorArgs: unknown[] = []; + vi.mocked(WebSocket).mockImplementation(function (this: unknown, ...args: unknown[]) { + constructorArgs = args; + return mockWs as unknown as WebSocket; + } as unknown as typeof WebSocket); - it("returns null for empty string input", () => { - const result = parseShellMessage(""); + const logger = mock(); + const config = { PROVIDER_PROXY_URL: PROXY_URL } as DeploymentConfig; + const service = new ShellExecService(config, logger); - expect(result).toBeNull(); - }); - }); + return { service, mockWs, logger, getConstructorArgs: () => constructorArgs }; + } - function createShellExecInput( - overrides?: Partial<{ command: string; timeout: number; providerBaseUrl: string; gseq: number; oseq: number; service: string; dseq: string }> - ): Parameters[0] { + function createShellExecInput(overrides?: Partial): ShellExecInput { return { providerBaseUrl: overrides?.providerBaseUrl ?? "https://provider.example.com", - providerAddress: faker.string.alphanumeric(44), - dseq: overrides?.dseq ?? "test-dseq", + providerAddress: overrides?.providerAddress ?? faker.string.alphanumeric(44), + dseq: overrides?.dseq ?? "1234", gseq: overrides?.gseq ?? 1, oseq: overrides?.oseq ?? 1, service: overrides?.service ?? "test-service", - command: overrides?.command ?? "echo Hello", + command: overrides?.command ?? ["echo", "Hello"], + stdin: overrides?.stdin, timeout: overrides?.timeout ?? 60, - jwtToken: faker.string.alphanumeric(100) + jwtToken: overrides?.jwtToken ?? faker.string.alphanumeric(100) }; } - describe("buildShellUrl edge cases - undefined/null providerBaseUrl", () => { - it("throws TypeError when providerBaseUrl is undefined (replace on undefined)", () => { - const input = { - providerBaseUrl: undefined as any, - dseq: "test-dseq", - gseq: 1, - oseq: 1, - service: "test-service", - command: "echo hello" - }; - - expect(() => buildShellUrl(input)).toThrow(TypeError); - }); - - it("throws TypeError when providerBaseUrl is null (replace on null)", () => { - const input = { - providerBaseUrl: null as any, - dseq: "test-dseq", - gseq: 1, - oseq: 1, - service: "test-service", - command: "echo hello" - }; - - expect(() => buildShellUrl(input)).toThrow(TypeError); - }); - }); - - describe("buildShellUrl edge cases - undefined inputs", () => { - it("throws TypeError when command is undefined (trim on undefined)", () => { - const input = { - providerBaseUrl: "https://provider.example.com", - dseq: "test-dseq", - gseq: 1, - oseq: 1, - service: "test-service", - command: undefined as any - }; - - expect(() => buildShellUrl(input)).toThrow(TypeError); - }); - - it("throws TypeError when command is null (trim on null)", () => { - const input = { - providerBaseUrl: "https://provider.example.com", - dseq: "test-dseq", - gseq: 1, - oseq: 1, - service: "test-service", - command: null as any - }; - - expect(() => buildShellUrl(input)).toThrow(TypeError); - }); - - it("encodes undefined dseq as literal string 'undefined' in URL path", () => { - const input = { - providerBaseUrl: "https://provider.example.com", - dseq: undefined as any, - gseq: 1, - oseq: 1, - service: "test-service", - command: "echo hello" - }; - - const url = buildShellUrl(input); - - expect(url).toContain("/lease/undefined/1/1/shell"); - }); - - it("encodes undefined service as literal string 'undefined'", () => { - const input = { - providerBaseUrl: "https://provider.example.com", - dseq: "test-dseq", - gseq: 1, - oseq: 1, - service: undefined as any, - command: "echo hello" - }; - - const url = buildShellUrl(input); - - expect(url).toContain("service=undefined"); - }); - }); - - describe("parseShellMessage edge cases - null and object message types", () => { - it("returns null when message.message is null (binary path)", () => { - const result = parseShellMessage("[1, null]"); - - expect(result).toBeNull(); - }); - - it("returns null when message.message is an object instead of string/array", () => { - const result = parseShellMessage('{"type": "websocket", "message": {}}'); - - expect(result).toBeNull(); - }); - - it("returns null when message.message is a number instead of string/array", () => { - const result = parseShellMessage('{"type": "websocket", "message": 123}'); - - expect(result).toBeNull(); - }); - }); - - describe("binary message parsing - stream marker 0 with payload", () => { - it("binary message [0, 72, 101] with stream marker 0 AND payload bytes should be ignored (exit code not set)", () => { - const result = parseShellMessage("[0, 72, 101, 108, 108, 111]"); - - expect(result).toBeNull(); - }); - - it("binary message [0, 0] with stream marker 0 AND null byte should be ignored", () => { - const result = parseShellMessage("[0, 0]"); - - expect(result).toBeNull(); - }); - - it("binary message [1, 256] with out-of-range byte should not crash (replacement character)", () => { - const result = parseShellMessage("[1, 256]"); - - expect(result).toBeNull(); - }); - - it("binary message [1, -1] with negative byte value should not crash", () => { - const result = parseShellMessage("[1, -1]"); - - expect(result).toBeNull(); - }); - }); - - describe("parseShellMessage - whitespace-only message string", () => { - it("returns data for message with only spaces (length > 0 check passes)", () => { - const result = parseShellMessage('{"message": " "}'); - - expect(result).toEqual({ type: "data", data: " ", stream: "stdout" }); - }); - - it("returns data for message with only tabs", () => { - const result = parseShellMessage('{"message": "\\t\\t"}'); - - expect(result).toEqual({ type: "data", data: "\t\t", stream: "stdout" }); - }); - - it("returns data for message containing only newlines (whitespace IS valid shell output)", () => { - const result = parseShellMessage('{"message": "\\n\\n\\n"}'); - - expect(result).toEqual({ type: "data", data: "\n\n\n", stream: "stdout" }); - }); - - it("returns data for message containing only carriage returns", () => { - const result = parseShellMessage('{"message": "\\r\\r\\r"}'); - - expect(result).toEqual({ type: "data", data: "\r\r\r", stream: "stdout" }); - }); - - it("returns data for message containing mixed whitespace (space, newline, tab)", () => { - const result = parseShellMessage('{"message": " \\n\\t "}'); - - expect(result).toEqual({ type: "data", data: " \n\t ", stream: "stdout" }); - }); - }); - - describe("parseShellMessage - binary stderr stream marker 2", () => { - it("binary message [2, 72, 101, 108, 108, 111] with stderr stream marker returns null (binary path)", () => { - const result = parseShellMessage("[2, 72, 101, 108, 108, 111]"); - - expect(result).toBeNull(); - }); - - it("binary message [2] with stderr stream marker and no payload returns null", () => { - const result = parseShellMessage("[2]"); - - expect(result).toBeNull(); - }); - }); - - describe("execute - message.error field edge cases", () => { - function createMockWebSocket() { - const handlers: Record void>> = {}; - return { - on: vi.fn((event: string, handler: (...args: any[]) => void) => { - if (!handlers[event]) handlers[event] = []; - handlers[event].push(handler); - }), - send: vi.fn(), - close: vi.fn(), - _trigger(event: string, ...args: any[]) { - (handlers[event] || []).forEach(h => h(...args)); - } - }; - } - - it("message.error truthy string returns Err with provider error", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "some data"}' }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "error", error: "provider connection failed" }))); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toContain("Provider error: provider connection failed"); - }); - - it("WebSocket error event returns Err with connection failure message", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "partial"}' }))); - mockWs._trigger("error", new Error("ECONNREFUSED")); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toContain("WebSocket connection failed: ECONNREFUSED"); - }); - }); - - describe("execute - timeout + close double resolve", () => { - function createMockWebSocket() { - const handlers: Record void>> = {}; - return { - on: vi.fn((event: string, handler: (...args: any[]) => void) => { - if (!handlers[event]) handlers[event] = []; - handlers[event].push(handler); - }), - send: vi.fn(), - close: vi.fn(), - _trigger(event: string, ...args: any[]) { - (handlers[event] || []).forEach(h => h(...args)); - } - }; - } - - it("timeout resolving then close event firing causes double resolve (first Err should win)", async () => { - vi.useFakeTimers(); - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: 5 })); - - mockWs._trigger("open"); - - vi.advanceTimersByTime(5000); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toBe("Command timed out"); - - vi.useRealTimers(); - }); - - it("negative timeout fires immediately returning timeout error", async () => { - vi.useFakeTimers(); - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: -1 })); - - mockWs._trigger("open"); - - vi.advanceTimersByTime(0); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toBe("Command timed out"); - - vi.useRealTimers(); - }); - }); - - describe("execute - string message parse failure silently dropped", () => { - function createMockWebSocket() { - const handlers: Record void>> = {}; - return { - on: vi.fn((event: string, handler: (...args: any[]) => void) => { - if (!handlers[event]) handlers[event] = []; - handlers[event].push(handler); - }), - send: vi.fn(), - close: vi.fn(), - _trigger(event: string, ...args: any[]) { - (handlers[event] || []).forEach(h => h(...args)); - } - }; - } - - it("string message that fails parseShellMessage is silently dropped (not accumulated)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "first" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: "this is not JSON so parseShellMessage returns null" }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "third" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("firstthird"); - }); - }); - - describe("execute - closed message after exitCode already set", () => { - function createMockWebSocket() { - const handlers: Record void>> = {}; - return { - on: vi.fn((event: string, handler: (...args: any[]) => void) => { - if (!handlers[event]) handlers[event] = []; - handlers[event].push(handler); - }), - send: vi.fn(), - close: vi.fn(), - _trigger(event: string, ...args: any[]) { - (handlers[event] || []).forEach(h => h(...args)); - } - }; - } - - it("message.closed true arrives after exit_code 0 - exitCode stays 0 (first exit_code wins)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - }); - - describe("buildShellUrl - command with multiple consecutive spaces", () => { - it("command with three consecutive spaces between words splits correctly", () => { - const input = createShellExecInput({ command: "echo hello world" }); - - const url = buildShellUrl(input); - - expect(url).toContain("cmd0=echo"); - expect(url).toContain("cmd1=hello"); - expect(url).toContain("cmd2=world"); - expect(url).not.toContain("cmd3="); - }); - }); - - describe("execute - WebSocket integration edge cases", () => { - function createMockWebSocket() { - const handlers: Record void>> = {}; - return { - on: vi.fn((event: string, handler: (...args: any[]) => void) => { - if (!handlers[event]) handlers[event] = []; - handlers[event].push(handler); - }), - send: vi.fn(), - close: vi.fn(), - _trigger(event: string, ...args: any[]) { - (handlers[event] || []).forEach(h => h(...args)); - } - }; - } - - it("discards binary payload with stream marker 0 instead of routing to stdout or stderr", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: [0, 72, 101, 108, 108, 111] - }) - ) - ); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe(""); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("allows combined binary stdout+stderr to exceed MAX_OUTPUT_SIZE by checking each stream independently", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - - const halfSize = 524288; - const stdoutData = new Array(halfSize + 1).fill(65); - stdoutData[0] = 1; - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: stdoutData }))); - - const stderrData = new Array(halfSize + 2).fill(66); - stderrData[0] = 2; - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: stderrData }))); - - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [0] }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).truncated).toBe(true); - }); - - it("binary stream marker 1 (stdout) appends data to stdout and combines with string messages", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "string-" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [1, 66, 105, 110, 97, 114, 121] }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("string-Binary"); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("unknown message type does not corrupt stdout accumulation", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "first" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "unknown", data: "corrupt" }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "second" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("firstsecond"); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("binary message with stream marker 1 and no payload is ignored (length check)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: JSON.stringify({ message: "data" }) }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [1] }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("data"); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("leaves promise unresolved when error event fires without close event", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: 30 })); - - mockWs._trigger("error", new Error("connection failed")); - - const raceResult = await Promise.race([ - promise.then(r => ({ status: "resolved" as const, result: r })), - new Promise<{ status: "pending" }>(resolve => setTimeout(() => resolve({ status: "pending" }), 100)) - ]); - - mockWs._trigger("close"); - - expect(raceResult.status).toBe("resolved"); - }); - - it("error event resolves before close event can fire (first resolve wins)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("error", new Error("connection failed")); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toContain("WebSocket connection failed: connection failed"); - }); - - it("discards accumulated stdout data when timeout fires returning only error string", async () => { - vi.useFakeTimers(); - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: 5 })); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"message": "partial output"}' }))); - - vi.advanceTimersByTime(5000); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toBe("Command timed out"); - - vi.useRealTimers(); - }); - - it("defaults exitCode to 1 when message.closed arrives with no prior exit_code", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).exitCode).toBe(1); - }); - - it("uses last exit_code when multiple exit_code JSON messages arrive sequentially", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 42}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).exitCode).toBe(42); - }); - - it("string message path falls back to message when exit_code is non-numeric string", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const shellExecService = new ShellExecService(); - const promise = shellExecService.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ exit_code: "invalid", message: "fallback data" }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("fallback data"); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("resolves with timeout error immediately when timeout is 0 (setTimeout 0ms)", async () => { - vi.useFakeTimers(); - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: 0 })); - - vi.advanceTimersByTime(0); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toBe("Command timed out"); - expect(mockWs.close).toHaveBeenCalled(); - - vi.useRealTimers(); - }); - - it("resolves with error when WebSocket closes without ever receiving exit code", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(false); - expect(result.val).toBe("Connection closed without exit code"); - }); - - it("message arriving before timeout does not cause double resolve when timeout fires and close event also fires", async () => { - vi.useFakeTimers(); - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput({ timeout: 5 })); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - - vi.advanceTimersByTime(5000); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - - vi.useRealTimers(); - }); - - it("message.closed true followed by close event does not double-resolve (closed sets exitCode=1, close resolves)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger("message", Buffer.from(JSON.stringify({ closed: true }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).exitCode).toBe(1); - }); - - it("string message path never routes to stderr even with error-like content (parseShellMessage always returns stream: stdout)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ message: "error: something went wrong" }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("error: something went wrong"); - expect((result.val as ShellExecOutput).stderr).toBe(""); - }); - - it("binary payload with firstByte 3-255 is silently discarded (dead zone in protocol)", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: [3, 72, 101, 108, 108, 111] - }) - ) - ); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: [255, 87, 111, 114, 108, 100] - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: [0] }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe(""); - expect((result.val as ShellExecOutput).stderr).toBe(""); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - - it("output at exactly MAX_OUTPUT_SIZE (1048576 bytes) is NOT truncated", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - const exactSize = 1024 * 1024; - const data = "A".repeat(exactSize); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ message: data }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).truncated).toBe(false); - expect((result.val as ShellExecOutput).stdout).toBe(data); - expect((result.val as ShellExecOutput).stdout.length).toBe(1048576); - }); - - it("output exceeding MAX_OUTPUT_SIZE by 1 byte (1048577) IS truncated and data is discarded", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); - - mockWs._trigger("open"); - const overSize = 1024 * 1024 + 1; - const data = "A".repeat(overSize); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ message: data }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); - - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).truncated).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe(""); - }); - - it("pong messages interspersed with data messages do not corrupt output accumulation", async () => { - const mockWs = createMockWebSocket(); - vi.mocked(WebSocket).mockImplementation(function (this: any) { - return mockWs; - }); - - const service = new ShellExecService(); - const promise = service.execute(createShellExecInput()); + // Provider -> proxy -> client frame shape: { type:"websocket", message:{ type:"Buffer", data:number[] } } + function bytesFrame(data: number[]): Buffer { + return Buffer.from(JSON.stringify({ type: "websocket", message: { type: "Buffer", data } })); + } - mockWs._trigger("open"); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ message: "Hello " }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "pong" }))); - mockWs._trigger( - "message", - Buffer.from( - JSON.stringify({ - type: "websocket", - message: JSON.stringify({ message: "World" }) - }) - ) - ); - mockWs._trigger("message", Buffer.from(JSON.stringify({ type: "websocket", message: '{"exit_code": 0}' }))); - mockWs._trigger("close"); + function dataFrame(marker: number, text: string): Buffer { + return bytesFrame([marker, ...Buffer.from(text, "utf-8")]); + } - const result = await promise; - expect(result.ok).toBe(true); - expect((result.val as ShellExecOutput).stdout).toBe("Hello World"); - expect((result.val as ShellExecOutput).exitCode).toBe(0); - }); - }); + function exitFrameJson(code: number): Buffer { + return bytesFrame([102, ...Buffer.from(JSON.stringify({ exit_code: code }), "utf-8")]); + } }); diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts index 461860a488..007f8c15cb 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts @@ -1,9 +1,26 @@ +import { isIP } from "node:net"; import { Err, Ok, Result } from "ts-results"; -import { singleton } from "tsyringe"; +import { inject, singleton } from "tsyringe"; import { WebSocket } from "ws"; +import { LoggerService } from "@src/core"; +import { DEPLOYMENT_CONFIG, type DeploymentConfig } from "@src/deployment/config/config.provider"; + const MAX_OUTPUT_SIZE = 1024 * 1024; +/** + * LeaseShell protocol markers. The provider prefixes every shell frame with a + * single marker byte, mirroring `LeaseShellCode` in the web client + * (`apps/deploy-web/src/types/shell.ts`). + */ +const LeaseShellCode = { + Stdout: 100, + Stderr: 101, + Result: 102, + Failure: 103, + Stdin: 104 +} as const; + export type ShellExecInput = { providerBaseUrl: string; providerAddress: string; @@ -11,9 +28,15 @@ export type ShellExecInput = { gseq: number; oseq: number; service: string; - command: string; + command: string[]; timeout: number; jwtToken: string; + /** + * Optional data piped to the command's standard input. Kept out of the URL/argv + * (which the provider-proxy logs) and streamed as `104` LeaseShellCodeStdin + * frames instead, so secrets never touch the logged request URL. + */ + stdin?: string; }; export type ShellExecOutput = { @@ -23,262 +46,313 @@ export type ShellExecOutput = { truncated: boolean; }; +type ProviderBufferMessage = { + type?: string; + data?: number[] | string; +}; + +type ReceivedMessage = { + type?: string; + message?: string | number[] | ProviderBufferMessage | null; + closed?: boolean; + code?: number; + reason?: string | number[] | { data?: number[] } | null; + error?: unknown; + errors?: Array<{ path?: Array; message?: string }>; +}; + +type WebSocketOutgoingMessage = { + type: "websocket"; + url: string; + auth: { type: "jwt"; token: string }; + providerAddress: string; + isBase64: boolean; +}; + +// A post-connect frame carries a base64 `data` payload alongside the full +// envelope. The provider-proxy re-validates the envelope on every frame, so the +// url/auth/providerAddress fields must be resent, not just `data`. +type WebSocketDataMessage = WebSocketOutgoingMessage & { data: string }; + @singleton() export class ShellExecService { - constructor() {} + private readonly proxyWsUrl: string; + + constructor( + @inject(DEPLOYMENT_CONFIG) config: DeploymentConfig, + private readonly logger: LoggerService + ) { + this.proxyWsUrl = toProxyWebSocketUrl(config.PROVIDER_PROXY_URL); + this.logger.setContext(ShellExecService.name); + } async execute(input: ShellExecInput): Promise> { - const url = buildShellUrl(input); + if (!isValidProviderHost(input.providerBaseUrl)) { + return Err(`Invalid provider host: ${input.providerBaseUrl}`); + } + + const providerUrl = buildShellUrl(input); const auth = { type: "jwt" as const, token: input.jwtToken }; return new Promise(resolve => { - const timeoutId = setTimeout(() => { - ws.close(); - resolve(Err("Command timed out")); - }, input.timeout * 1000); + let settled = false; + const settle = (result: Result) => { + if (settled) return; + settled = true; + clearTimeout(timeoutId); + resolve(result); + }; let stdout = ""; let stderr = ""; let exitCode: number | undefined; let truncated = false; - const ws = new WebSocket(url, { - headers: { Authorization: `Bearer ${auth.token}` } - }); + // Connect to the provider-proxy (NOT the provider): no Authorization header, + // the provider credentials travel inside the envelope instead. + const ws = new WebSocket(this.proxyWsUrl); + + const timeoutId = setTimeout(() => { + settle(Err("Command timed out")); + ws.close(); + }, input.timeout * 1000); ws.on("open", () => { + // Connect frame. Omit `data`: with `stdin=0` the command runs one-shot + // from the URL query params, and the proxy's `!message.data` guard still + // opens and links the provider socket. const message: WebSocketOutgoingMessage = { type: "websocket", - url, + url: providerUrl, auth, providerAddress: input.providerAddress, isBase64: true }; ws.send(JSON.stringify(message)); + + // Task 2b: stream any stdin as `104` LeaseShellCodeStdin frames instead + // of embedding secrets in the (logged) URL/argv. The provider-proxy + // queues data frames until the provider socket is verified, so no fixed + // settle delay is needed here. Each frame resends the full envelope + // (url/auth/providerAddress) with a base64 `data` payload. + if (input.stdin && input.stdin.length > 0) { + const sendData = (payload: Buffer) => { + const dataMessage: WebSocketDataMessage = { ...message, data: payload.toString("base64") }; + ws.send(JSON.stringify(dataMessage)); + }; + // Marker byte 104 prefixes the raw UTF-8 stdin bytes. + sendData(Buffer.concat([Buffer.from([LeaseShellCode.Stdin]), Buffer.from(input.stdin, "utf-8")])); + // EOF: a bare `104` marker with no payload closes stdin so the command + // (e.g. `cat`) sees end-of-input and can exit. + sendData(Buffer.from([LeaseShellCode.Stdin])); + } }); - ws.on("message", (data: Buffer) => { + const appendOutput = (marker: number, text: string) => { + if (truncated) return; + if (stdout.length + stderr.length + text.length <= MAX_OUTPUT_SIZE) { + if (marker === LeaseShellCode.Stdout) { + stdout += text; + } else { + stderr += text; + } + } else { + // M5: stop appending once over the cap, but keep reading so the + // exit/close frame still arrives and `exitCode` stays accurate. + truncated = true; + } + }; + + const handleClose = (code?: number, reason?: string) => { + if (isAuthExpiryClose(code, reason)) { + settle(Err(`Auth expired: provider closed connection${code !== undefined ? ` (code ${code})` : ""}`)); + return; + } + if (exitCode !== undefined) { + settle(Ok({ stdout, stderr, exitCode, truncated })); + } else { + settle(Err("Connection closed without exit code")); + } + }; + + ws.on("message", (raw: Buffer) => { + let message: ReceivedMessage; try { - const message = JSON.parse(data.toString()) as ReceivedMessage; + message = JSON.parse(raw.toString()) as ReceivedMessage; + } catch { + // Non-JSON frame: ignore. + return; + } + + // 1. Filter keepalives and unknown frame types. + if (message.type === "pong") return; + if (message.type !== undefined && message.type !== "websocket") return; + + // 2. Error BEFORE decode (M1). The proxy reports failures as a + // `type:"websocket"` frame carrying an `error` key — never decode it. + if (message.error !== undefined && message.error !== null && message.error !== "") { + settle(Err(`Provider error: ${flattenError(message)}`)); + ws.close(); + return; + } + + if (message.closed) { + handleClose(message.code, reasonToString(message.reason)); + ws.close(); + return; + } + + // 3. Decode payload. `message.message.data` is either a byte list (Node + // Buffer -> JSON) or a base64 string. + const inner = message.message; + const rawData = Array.isArray(inner) ? inner : inner && typeof inner === "object" ? (inner as ProviderBufferMessage).data : undefined; - if (message.type === "pong") { + let bytes: Buffer; + if (Array.isArray(rawData)) { + bytes = Buffer.from(rawData); + } else if (typeof rawData === "string") { + if (!isStrictBase64(rawData)) { + // Never dispatch a non-base64 prose frame as fake output. + this.logger.warn({ event: "SHELL_EXEC_INVALID_BASE64", length: rawData.length }); return; } + bytes = Buffer.from(rawData, "base64"); + } else { + return; + } - if (message.type === "websocket" && message.message) { - if (typeof message.message === "string") { - const parsed = parseShellMessage(message.message); - if (parsed) { - if (parsed.type === "exit_code") { - exitCode = parsed.exit_code; - clearTimeout(timeoutId); - ws.close(); - } else if (parsed.type === "data" && parsed.data) { - if (truncated) { - return; - } - const output = parsed.data; - if (output.length + stdout.length + stderr.length <= MAX_OUTPUT_SIZE) { - if (parsed.stream === "stdout") { - stdout += output; - } else { - stderr += output; - } - } else { - truncated = true; - } - } - } - } else if (Array.isArray(message.message)) { - const messageData = message.message; - if (messageData.length > 1) { - const firstByte = messageData[0]; - const payload = messageData.slice(1); - const textDecoder = new TextDecoder("utf-8"); - const output = textDecoder.decode(Buffer.from(payload)); - - if (firstByte === 0) { - exitCode = 0; - clearTimeout(timeoutId); - ws.close(); - } else if (firstByte === 1) { - if (truncated) { - return; - } - if (output.length + stdout.length + stderr.length <= MAX_OUTPUT_SIZE) { - stdout += output; - } else { - truncated = true; - } - } else if (firstByte === 2) { - if (truncated) { - return; - } - if (output.length + stderr.length + stdout.length <= MAX_OUTPUT_SIZE) { - stderr += output; - } else { - truncated = true; - } - } - } - - if (messageData.length === 1 && messageData[0] === 0) { - exitCode = 0; - clearTimeout(timeoutId); - ws.close(); - } - } - } + if (bytes.length === 0) return; + + // 4. Marker dispatch (M2). marker = data[0]; payload = data.slice(1). + const marker = bytes[0]; + const payload = bytes.subarray(1); - if (message.closed || message.error) { - clearTimeout(timeoutId); - if (exitCode === undefined) { - if (message.error) { - resolve(Err(`Provider error: ${message.error}`)); - } else { - exitCode = 1; - } - } + switch (marker) { + case LeaseShellCode.Stdout: + case LeaseShellCode.Stderr: + appendOutput(marker, payload.toString("utf-8")); + return; + case LeaseShellCode.Result: + // M3: exit code is either JSON `{"exit_code":N}` or a 4-byte LE int32. + exitCode = parseExitCode(payload); + settle(Ok({ stdout, stderr, exitCode, truncated })); + ws.close(); + return; + case LeaseShellCode.Failure: { + // M2: distinct provider-error end state, not output. + const detail = payload.toString("utf-8").trim(); + settle(Err(`Provider error: shell failure${detail ? `: ${detail}` : ""}`)); ws.close(); + return; } - } catch { - // Ignore parse errors + default: + // Unknown marker (e.g. stdin/resize echoes) — ignore. + return; } }); ws.on("error", err => { - clearTimeout(timeoutId); + settle(Err(`WebSocket connection failed: ${err.message}`)); ws.close(); - resolve(Err(`WebSocket connection failed: ${err.message}`)); }); - ws.on("close", () => { - clearTimeout(timeoutId); - if (exitCode !== undefined) { - resolve( - Ok({ - stdout, - stderr, - exitCode, - truncated - }) - ); - } else { - resolve(Err("Connection closed without exit code")); - } + ws.on("close", (code?: number, reason?: Buffer) => { + handleClose(code, reason ? reason.toString() : undefined); }); }); } } +/** Maps an http(s) proxy URL to its ws(s) equivalent. */ +export function toProxyWebSocketUrl(url: string): string { + return url.replace(/^http/, "ws"); +} + /** - * Tokenizes a shell command into argv-style tokens the way a POSIX shell splits - * a command line, so each token maps to one `cmdN` query param consumed by the - * provider's lease-shell endpoint (which execs the tokens as argv, with no shell - * re-interpretation of its own). - * - * A naive `command.split(" ")` corrupts any argument that legitimately contains - * whitespace — quoted strings, paths, JSON blobs, PEM bodies — and is especially - * broken for the post-deploy secret-injection use case (e.g. - * `sh -c "echo SECRET=v > /run/secrets/.env"`), which must survive as the three - * tokens `["sh", "-c", "echo SECRET=v > /run/secrets/.env"]`. - * - * Rules: unquoted whitespace delimits tokens (space/tab/newline — matching the - * POSIX default `IFS`, so a carriage return is preserved, not a delimiter); single quotes - * preserve their contents literally; double quotes preserve contents but allow - * `\` to escape `"` and `\`; a backslash outside quotes escapes the next char. - * Quote characters are removed from the emitted tokens (shell semantics). + * The provider-proxy silently rejects a lease-shell URL that is not https or + * that points at an IP / `.local` host. Pre-check so we can surface a clear + * error instead of a mystery timeout. */ -export function tokenizeCommand(command: string): string[] { - const tokens: string[] = []; - let current = ""; - let hasToken = false; - let quote: '"' | "'" | null = null; - - for (let i = 0; i < command.length; i++) { - const char = command[i]; - - if (quote === "'") { - if (char === "'") quote = null; - else current += char; - continue; - } - - if (quote === '"') { - if (char === "\\" && (command[i + 1] === '"' || command[i + 1] === "\\")) { - current += command[++i]; - } else if (char === '"') { - quote = null; - } else { - current += char; - } - continue; - } - - if (char === "'" || char === '"') { - quote = char; - hasToken = true; - continue; - } - - if (char === "\\" && i + 1 < command.length) { - current += command[++i]; - hasToken = true; - continue; - } - - if (char === " " || char === "\t" || char === "\n") { - if (hasToken) { - tokens.push(current); - current = ""; - hasToken = false; - } - continue; - } - - current += char; - hasToken = true; +export function isValidProviderHost(hostUri: string): boolean { + try { + const parsed = new URL(hostUri); + const hostname = parsed.hostname.startsWith("[") ? parsed.hostname.slice(1, -1) : parsed.hostname; + return parsed.protocol === "https:" && !hostname.endsWith(".local") && !isIP(hostname); + } catch { + return false; } - - if (hasToken) tokens.push(current); - return tokens; } -export function buildShellUrl(input: Pick): string { - const tokens = tokenizeCommand(input.command); - const cmdParts = tokens.length > 0 ? tokens.map((token, i) => `&cmd${i}=${encodeURIComponent(token)}`) : [`&cmd0=`]; +/** + * Builds the provider lease-shell URL consumed by the proxy as the envelope + * `url`. Each argv token maps to one `cmdN` query param; the provider execs the + * tokens as argv with no shell re-interpretation of its own. + */ +export function buildShellUrl(input: Pick): string { + const cmdParts = input.command.map((arg, i) => `&cmd${i}=${encodeURIComponent(arg)}`).join(""); const baseUrl = input.providerBaseUrl.replace(/\/$/, ""); - return `${baseUrl}/lease/${encodeURIComponent(input.dseq)}/${input.gseq}/${input.oseq}/shell?stdin=0&tty=0&podIndex=0&service=${encodeURIComponent(input.service)}${cmdParts.join("")}`; + // `stdin=1` opens the provider stdin channel so `104` frames are accepted; the + // payload itself is NEVER placed in the URL (it would be logged by the proxy). + const stdinFlag = input.stdin && input.stdin.length > 0 ? 1 : 0; + return `${baseUrl}/lease/${encodeURIComponent(input.dseq)}/${input.gseq}/${input.oseq}/shell?stdin=${stdinFlag}&tty=0&podIndex=0&service=${encodeURIComponent(input.service)}${cmdParts}`; } -export function parseShellMessage(message: string): { type: "data" | "exit_code"; data?: string; stream?: "stdout" | "stderr"; exit_code?: number } | null { - if (message.startsWith("{")) { +/** + * Parses the exit-code payload of a `102` result frame. Providers use one of two + * encodings (version-dependent): a JSON body `{"exit_code":N}` or a raw 4-byte + * little-endian int32. `null` and missing codes map to 0. + */ +export function parseExitCode(payload: Buffer): number { + const text = payload.toString("utf-8").trim(); + + if (text.startsWith("{")) { try { - const parsed = JSON.parse(message); - if (typeof parsed.exit_code === "number") { - return { type: "exit_code", exit_code: parsed.exit_code }; - } - if (typeof parsed.message === "string" && parsed.message.length > 0) { - return { type: "data", data: parsed.message, stream: "stdout" }; - } + const parsed = JSON.parse(text) as { exit_code?: unknown }; + const code = parsed?.exit_code; + // Successfully parsed JSON body: a null/absent/non-numeric code maps to 0. + return typeof code === "number" && Number.isFinite(code) ? code : 0; } catch { - return null; + // Malformed JSON that only happens to start with "{" (e.g. a 4-byte LE + // int32 whose first byte is 0x7B) — fall through to the binary encoding. } } - return null; + + if (payload.length >= 4) return payload.readInt32LE(0); + return 0; } -type WebSocketOutgoingMessage = { - type: "websocket"; - url: string; - auth: { type: "jwt"; token: string }; - providerAddress: string; - isBase64: boolean; -}; +/** Strict base64 validation so prose frames are dropped, not decoded as output. */ +export function isStrictBase64(value: string): boolean { + if (value.length === 0 || value.length % 4 !== 0) return false; + return /^[A-Za-z0-9+/]*={0,2}$/.test(value); +} -type ReceivedMessage = { - type?: string; - message?: string | number[]; - closed?: boolean; - error?: string; -}; +function isAuthExpiryClose(code?: number, reason?: string): boolean { + if (code === 4001 || code === 4003) return true; + if (reason && /expired|unauthorized/i.test(reason)) return true; + return false; +} + +function reasonToString(reason: ReceivedMessage["reason"]): string | undefined { + if (reason === undefined || reason === null) return undefined; + if (typeof reason === "string") return reason; + if (Array.isArray(reason)) return Buffer.from(reason).toString("utf-8"); + if (Array.isArray(reason.data)) return Buffer.from(reason.data).toString("utf-8"); + return undefined; +} + +/** Flattens a proxy error frame, including any Zod-style `errors[].path: message`. */ +function flattenError(message: ReceivedMessage): string { + const base = typeof message.error === "string" ? message.error : JSON.stringify(message.error); + if (Array.isArray(message.errors) && message.errors.length > 0) { + const flat = message.errors + .map(issue => { + const path = Array.isArray(issue?.path) ? issue.path.join(".") : ""; + return path ? `${path}: ${issue?.message ?? ""}` : `${issue?.message ?? ""}`; + }) + .filter(Boolean) + .join("; "); + if (flat) return `${base} (${flat})`; + } + return base; +} From 09be47f8ecbf1ec85509d3678a7895cc2d9a85bf Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Tue, 14 Jul 2026 11:23:58 +0100 Subject: [PATCH 3/4] =?UTF-8?q?fix(deployment):=20address=20CodeRabbit=20?= =?UTF-8?q?=E2=80=94=20UTF-8=20byte=20limits=20+=20typed=20test=20mocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enforce the 1 MiB stdin cap and output-truncation limit in UTF-8 bytes (Buffer.byteLength) instead of UTF-16 code units. - Use vitest-mock-extended mock() in the controller spec instead of `as` casts. - Generate test secret values dynamically (no static secret strings in tests). Co-Authored-By: Claude Opus 4.8 --- .../shell-exec/shell-exec.controller.spec.ts | 24 ++++++++++--------- .../http-schemas/shell-exec.schema.ts | 14 +++++++---- .../shell-exec/shell-exec.service.spec.ts | 6 ++--- .../services/shell-exec/shell-exec.service.ts | 7 +++++- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts index 91e11076c9..ad575c3bd2 100644 --- a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts @@ -17,18 +17,18 @@ type Lease = DeploymentResponse["leases"][number]; type ProviderInfo = NonNullable>>; function createLease(overrides: Partial<{ gseq: number; oseq: number; provider: string; state: string }> = {}): Lease { - return { + return mock({ id: { owner: "akash1owner", dseq: "1234", gseq: overrides.gseq ?? 1, oseq: overrides.oseq ?? 1, provider: overrides.provider ?? "akash1provider", bseq: 0 }, state: overrides.state ?? "active", price: { denom: "uakt", amount: "100" }, created_at: "12345", closed_on: "0", status: null - } as unknown as Lease; + }); } function createDeployment(overrides: Partial<{ state: string; leases: Lease[] }> = {}): DeploymentResponse { - return { + return mock({ deployment: { id: { owner: "akash1owner", dseq: "1234" }, state: overrides.state ?? "active", @@ -47,17 +47,18 @@ function createDeployment(overrides: Partial<{ state: string; leases: Lease[] }> deposits: [] } } - } as unknown as DeploymentResponse; + }); } function createProviderInfo(overrides: Partial<{ hostUri: string }> = {}): ProviderInfo { - return { hostUri: overrides.hostUri ?? "https://provider.example.com" } as unknown as ProviderInfo; + return mock({ hostUri: overrides.hostUri ?? "https://provider.example.com" }); } describe(ShellExecController.name, () => { it("throws 404 when deployment not found", async () => { const { controller, deploymentReaderService } = setup(); - deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(undefined as unknown as DeploymentResponse); + // Simulate "deployment not found": clear the default stub so the lookup resolves undefined. + deploymentReaderService.findByUserIdAndDseq.mockReset(); const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); @@ -123,6 +124,7 @@ describe(ShellExecController.name, () => { it("forwards stdin through to the shell exec service", async () => { const { controller, shellExecService } = setup(); + const secretValue = Math.random().toString(36).substring(2); await controller.exec({ dseq: "1234", @@ -131,10 +133,10 @@ describe(ShellExecController.name, () => { command: ["sh", "-c", "cat > /run/secrets/.env"], service: "web", timeout: 60, - stdin: "SECRET=value" + stdin: `SECRET=${secretValue}` }); - expect(shellExecService.execute).toHaveBeenCalledWith(expect.objectContaining({ stdin: "SECRET=value" })); + expect(shellExecService.execute).toHaveBeenCalledWith(expect.objectContaining({ stdin: `SECRET=${secretValue}` })); }); it("throws 404 when provider info lookup returns null", async () => { @@ -252,9 +254,9 @@ describe(ShellExecController.name, () => { const deployment = createDeployment({ leases: [createLease({ provider, state })] }); deploymentReaderService.findByUserIdAndDseq.mockResolvedValue(deployment); - walletReaderService.getWalletByUserId.mockResolvedValue({ id: 1, address: "akash1wallet" } as Awaited< - ReturnType - >); + walletReaderService.getWalletByUserId.mockResolvedValue( + mock>>({ id: 1, address: "akash1wallet" }) + ); providerService.toProviderAuth.mockResolvedValue({ type: "jwt" as const, token: "test-token" }); providerService.getProvider.mockResolvedValue(createProviderInfo()); shellExecService.execute.mockResolvedValue(new Ok({ stdout: "output", stderr: "", exitCode: 0, truncated: false })); diff --git a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts index c58a612560..ec3fd90949 100644 --- a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts +++ b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts @@ -10,11 +10,15 @@ export const ShellExecRequestSchema = z.object({ command: z.array(z.string().min(1)).min(1).max(64), service: z.string().min(1).max(253), timeout: z.number().int().min(1).max(120).default(60), - stdin: z.string().max(1_048_576).optional().openapi({ - description: - 'Optional raw UTF-8 data streamed to the command\'s standard input (max 1 MiB). Put secrets HERE (env files, tokens, passwords) — never in `command`, whose tokens are placed in the provider-proxy request URL, which is logged. Example command: ["sh","-c","cat > /run/secrets/.env"].', - example: "SECRET=value" - }) + stdin: z + .string() + .refine(s => Buffer.byteLength(s, "utf8") <= 1_048_576, { message: "stdin must not exceed 1 MiB (UTF-8 bytes)" }) + .optional() + .openapi({ + description: + 'Optional raw UTF-8 data streamed to the command\'s standard input (max 1 MiB). Put secrets HERE (env files, tokens, passwords) — never in `command`, whose tokens are placed in the provider-proxy request URL, which is logged. Example command: ["sh","-c","cat > /run/secrets/.env"].', + example: "SECRET=value" + }) }); export const ShellExecResponseSchema = z.object({ diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts index 70a23542f3..cd85e3ee27 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts @@ -149,7 +149,7 @@ describe(ShellExecService.name, () => { }); it("emits stdin=1 when stdin is provided", () => { - const url = buildShellUrl(createShellExecInput({ stdin: "SECRET=topsecret" })); + const url = buildShellUrl(createShellExecInput({ stdin: `SECRET=${faker.string.alphanumeric(16)}` })); expect(url).toContain("stdin=1"); expect(url).not.toContain("stdin=0"); @@ -163,7 +163,7 @@ describe(ShellExecService.name, () => { }); it("never places the stdin payload in the URL", () => { - const secret = "SUPER_SECRET_VALUE_12345"; + const secret = `SUPER_SECRET_VALUE_${faker.string.alphanumeric(16)}`; const url = buildShellUrl(createShellExecInput({ command: ["sh", "-c", "cat > /run/secrets/.env"], stdin: secret })); expect(url).not.toContain(secret); @@ -252,7 +252,7 @@ describe(ShellExecService.name, () => { it("sends a 104 stdin data frame plus a 104 EOF frame, keeping the secret out of the URL", async () => { const { service, mockWs, getConstructorArgs } = createService(); - const secret = "SECRET=topsecret\nAPI_KEY=abc123"; + const secret = `SECRET=${faker.string.alphanumeric(16)}\nAPI_KEY=${faker.string.alphanumeric(12)}`; const input = createShellExecInput({ command: ["sh", "-c", "cat > /run/secrets/.env && chmod 600 /run/secrets/.env"], stdin: secret diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts index 007f8c15cb..062e0f6e87 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts @@ -105,6 +105,7 @@ export class ShellExecService { let stdout = ""; let stderr = ""; + let outputBytes = 0; let exitCode: number | undefined; let truncated = false; @@ -150,7 +151,11 @@ export class ShellExecService { const appendOutput = (marker: number, text: string) => { if (truncated) return; - if (stdout.length + stderr.length + text.length <= MAX_OUTPUT_SIZE) { + // Enforce the cap in UTF-8 bytes (not UTF-16 code units) so multi-byte + // output is accounted for by its true wire size. + const chunkBytes = Buffer.byteLength(text, "utf8"); + if (outputBytes + chunkBytes <= MAX_OUTPUT_SIZE) { + outputBytes += chunkBytes; if (marker === LeaseShellCode.Stdout) { stdout += text; } else { From 0157d43fc621ef384c1db9690cb11f04a7e7ce4a Mon Sep 17 00:00:00 2001 From: "open-swe[bot]" Date: Sat, 18 Jul 2026 20:27:42 +0100 Subject: [PATCH 4/4] fix(deployment): harden shell-exec DoS caps + error-status mapping (stalniy review) Addresses @stalniy's CHANGES_REQUESTED on #3097. Plan converged via a 3-round multi-model quorum (unanimous APPROVE; full-convergence, dry improvement stream). Memory/DoS (both blocking comments): - MAX_OUTPUT_SIZE 1 MiB -> 64 KiB (exported); a synchronous buffered response is only safe while the buffer stays small. - stdin 1 MiB -> 16 KiB; command bounded per-arg by UTF-8 BYTES (not z.string().max, which counts UTF-16 code units) at 64 args x 1 KiB. - All request caps exported as named constants; SHELL_EXEC_BODY_LIMIT_BYTES is DERIVED arithmetically from them and wired into the route bodyLimit so the body limit and schema caps cannot drift. Error mapping: - Replace the 6-level nested ternary with a pure mapExecError() lookup table with correct statuses: timeout 504, auth-expired 403, invalid-provider-host 502 (hostUri is server-derived on-chain data, not client input, so 400 misattributes blame), provider/connection 502 with distinct messages. Router documents 403/504/413. Tests: - Rename service-spec factory createService() -> setup() per repo convention. - Truncation tests assert against the exported MAX_OUTPUT_SIZE. - New schema spec: stdin/command byte caps incl. multi-byte (emoji) case; body-limit derivation. - mapExecError contract test per error prefix -> status. 85 shell-exec unit tests pass; tsc (shell-exec) + eslint clean. Co-Authored-By: Claude Opus 4.8 --- .../shell-exec/shell-exec.controller.spec.ts | 54 +++++++++++-- .../shell-exec/shell-exec.controller.ts | 43 +++++++--- .../http-schemas/shell-exec.schema.spec.ts | 79 +++++++++++++++++++ .../http-schemas/shell-exec.schema.ts | 52 ++++++++++-- .../routes/shell-exec/shell-exec.router.ts | 20 ++++- .../shell-exec/shell-exec.service.spec.ts | 57 ++++++------- .../services/shell-exec/shell-exec.service.ts | 9 ++- 7 files changed, 260 insertions(+), 54 deletions(-) create mode 100644 apps/api/src/deployment/http-schemas/shell-exec.schema.spec.ts diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts index ad575c3bd2..5e908a3fb7 100644 --- a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.spec.ts @@ -8,7 +8,7 @@ import type { WalletReaderService } from "@src/billing/services/wallet-reader/wa import type { DeploymentReaderService } from "@src/deployment/services/deployment-reader/deployment-reader.service"; import type { ShellExecService } from "@src/deployment/services/shell-exec/shell-exec.service"; import type { ProviderService } from "@src/provider/services/provider/provider.service"; -import { ShellExecController } from "./shell-exec.controller"; +import { mapExecError, ShellExecController } from "./shell-exec.controller"; import { createUser } from "@test/seeders/user.seeder"; @@ -54,6 +54,20 @@ function createProviderInfo(overrides: Partial<{ hostUri: string }> = {}): Provi return mock({ hostUri: overrides.hostUri ?? "https://provider.example.com" }); } +describe("mapExecError", () => { + it.each([ + ["Command timed out", 504, "Command execution timed out"], + ["Auth expired: provider closed connection (code 4001)", 403, "Provider authentication expired"], + ["Invalid provider host: http://provider.example.com", 502, "Invalid provider host"], + ["WebSocket connection failed: ECONNREFUSED", 502, "Failed to connect to provider"], + ["Provider error: internal error", 502, "Provider returned an error"], + ["Connection closed without exit code", 502, "Provider connection closed unexpectedly"], + ["some entirely unexpected failure", 502, "Shell execution failed"] + ])("maps %j to status %i", (errVal, status, message) => { + expect(mapExecError(errVal as string)).toEqual({ status, message }); + }); +}); + describe(ShellExecController.name, () => { it("throws 404 when deployment not found", async () => { const { controller, deploymentReaderService } = setup(); @@ -93,13 +107,13 @@ describe(ShellExecController.name, () => { expect(error.message).toBe("Lease is not active"); }); - it("throws 502 when shell exec service returns an error result", async () => { + it("throws 504 when the command times out", async () => { const { controller, shellExecService } = setup(); shellExecService.execute.mockResolvedValue(Err("Command timed out")); const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); - expect(error.status).toBe(502); + expect(error.status).toBe(504); expect(error.message).toBe("Command execution timed out"); }); @@ -217,16 +231,46 @@ describe(ShellExecController.name, () => { expect(error.message).toBe("Provider returned an error"); }); - it("throws 502 with auth-expired message when the provider JWT expires mid-run", async () => { + it("throws 403 when the provider JWT expires mid-run", async () => { const { controller, shellExecService } = setup(); shellExecService.execute.mockResolvedValue(Err("Auth expired: provider closed connection (code 4001)")); const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); - expect(error.status).toBe(502); + expect(error.status).toBe(403); expect(error.message).toBe("Provider authentication expired"); }); + it("throws 502 for an invalid (server-derived) provider host", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("Invalid provider host: http://provider.example.com")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Invalid provider host"); + }); + + it("throws 502 with a distinct message when the connection closes without an exit code", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("Connection closed without exit code")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Provider connection closed unexpectedly"); + }); + + it("falls back to a generic 502 for an unrecognized service error", async () => { + const { controller, shellExecService } = setup(); + shellExecService.execute.mockResolvedValue(Err("some entirely unexpected failure")); + + const error = await captureError(() => controller.exec({ dseq: "1234", gseq: 1, oseq: 1, command: ["ls"], service: "web", timeout: 60 })); + + expect(error.status).toBe(502); + expect(error.message).toBe("Shell execution failed"); + }); + async function captureError(fn: () => Promise): Promise<{ status: number; message: string }> { try { await fn(); diff --git a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts index b723778e54..ce9f39d156 100644 --- a/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts +++ b/apps/api/src/deployment/controllers/shell-exec/shell-exec.controller.ts @@ -8,6 +8,35 @@ import { DeploymentReaderService } from "@src/deployment/services/deployment-rea import { ShellExecService } from "@src/deployment/services/shell-exec/shell-exec.service"; import { ProviderService } from "@src/provider/services/provider/provider.service"; +type ExecErrorMapping = { status: number; message: string }; + +/** + * Ordered mapping from a service-layer error sentinel (the stable, prefixed + * strings returned by `ShellExecService.execute`) to a user-facing HTTP status + * and message. Keeping this a table — rather than a chain of ternaries — makes + * the contract readable, testable, and cheap to extend, and guarantees the raw + * internal string is never leaked to the client. Each failure mode gets the + * semantically correct status: + * - timeout → 504 (we are a gateway to the provider, not a slow client) + * - auth expired → 403 + * - invalid host → 502 (`hostUri` is server-derived on-chain data, not + * client input, so a bad host is an upstream fault) + * - connection/provider→ 502 + */ +const EXEC_ERROR_TABLE: ReadonlyArray<{ prefix: string } & ExecErrorMapping> = [ + { prefix: "Command timed out", status: 504, message: "Command execution timed out" }, + { prefix: "Auth expired", status: 403, message: "Provider authentication expired" }, + { prefix: "Invalid provider host", status: 502, message: "Invalid provider host" }, + { prefix: "WebSocket connection failed", status: 502, message: "Failed to connect to provider" }, + { prefix: "Provider error", status: 502, message: "Provider returned an error" }, + { prefix: "Connection closed without exit code", status: 502, message: "Provider connection closed unexpectedly" } +]; + +export function mapExecError(errVal: string): ExecErrorMapping { + const match = EXEC_ERROR_TABLE.find(entry => errVal.startsWith(entry.prefix)); + return match ? { status: match.status, message: match.message } : { status: 502, message: "Shell execution failed" }; +} + @singleton() export class ShellExecController { constructor( @@ -57,18 +86,8 @@ export class ShellExecController { }); if (!result.ok) { - const message = result.val.startsWith("Command timed out") - ? "Command execution timed out" - : result.val.startsWith("WebSocket connection failed") - ? "Failed to connect to provider" - : result.val.startsWith("Auth expired") - ? "Provider authentication expired" - : result.val.startsWith("Invalid provider host") - ? "Invalid provider host" - : result.val.startsWith("Provider error") - ? "Provider returned an error" - : "Shell execution failed"; - assert(false, 502, message); + const { status, message } = mapExecError(result.val); + assert(false, status, message); } return result.val; diff --git a/apps/api/src/deployment/http-schemas/shell-exec.schema.spec.ts b/apps/api/src/deployment/http-schemas/shell-exec.schema.spec.ts new file mode 100644 index 0000000000..0bfce980dd --- /dev/null +++ b/apps/api/src/deployment/http-schemas/shell-exec.schema.spec.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { + JSON_ENVELOPE_OVERHEAD_BYTES, + MAX_COMMAND_ARG_BYTES, + MAX_COMMAND_ARGS, + MAX_STDIN_BYTES, + SERVICE_NAME_MAX, + SHELL_EXEC_BODY_LIMIT_BYTES, + ShellExecRequestSchema +} from "./shell-exec.schema"; + +describe("ShellExecRequestSchema", () => { + describe("stdin byte cap", () => { + it("accepts stdin at exactly the byte cap", () => { + const result = ShellExecRequestSchema.safeParse(base({ stdin: "A".repeat(MAX_STDIN_BYTES) })); + expect(result.success).toBe(true); + }); + + it("rejects stdin one byte over the cap", () => { + const result = ShellExecRequestSchema.safeParse(base({ stdin: "A".repeat(MAX_STDIN_BYTES + 1) })); + expect(result.success).toBe(false); + }); + + it("counts UTF-8 bytes, not UTF-16 code units, so a short multi-byte string over the byte cap is rejected", () => { + // "😀" is 2 UTF-16 code units but 4 UTF-8 bytes. This many are under MAX_STDIN_BYTES + // by `.length` (code units) yet over it in bytes — a `z.string().max()` would wrongly pass. + const emojiCount = Math.floor(MAX_STDIN_BYTES / 2); + const value = "😀".repeat(emojiCount); + expect(value.length).toBeLessThanOrEqual(MAX_STDIN_BYTES); + expect(Buffer.byteLength(value, "utf8")).toBeGreaterThan(MAX_STDIN_BYTES); + + const result = ShellExecRequestSchema.safeParse(base({ stdin: value })); + expect(result.success).toBe(false); + }); + }); + + describe("command caps", () => { + it("accepts a command arg at exactly the per-arg byte cap", () => { + const result = ShellExecRequestSchema.safeParse(base({ command: ["sh", "-c", "A".repeat(MAX_COMMAND_ARG_BYTES)] })); + expect(result.success).toBe(true); + }); + + it("rejects a command arg one byte over the per-arg cap", () => { + const result = ShellExecRequestSchema.safeParse(base({ command: ["sh", "-c", "A".repeat(MAX_COMMAND_ARG_BYTES + 1)] })); + expect(result.success).toBe(false); + }); + + it("rejects more than the maximum number of command args", () => { + const result = ShellExecRequestSchema.safeParse(base({ command: Array.from({ length: MAX_COMMAND_ARGS + 1 }, () => "x") })); + expect(result.success).toBe(false); + }); + + it("rejects an empty command array", () => { + const result = ShellExecRequestSchema.safeParse(base({ command: [] })); + expect(result.success).toBe(false); + }); + }); + + describe("SHELL_EXEC_BODY_LIMIT_BYTES", () => { + it("is derived arithmetically from the field caps (never hardcoded)", () => { + expect(SHELL_EXEC_BODY_LIMIT_BYTES).toBe(MAX_COMMAND_ARGS * MAX_COMMAND_ARG_BYTES + MAX_STDIN_BYTES + SERVICE_NAME_MAX + JSON_ENVELOPE_OVERHEAD_BYTES); + }); + + it("leaves headroom above the largest single field cap so a schema-valid body is not rejected first", () => { + expect(SHELL_EXEC_BODY_LIMIT_BYTES).toBeGreaterThan(MAX_STDIN_BYTES); + expect(SHELL_EXEC_BODY_LIMIT_BYTES).toBeGreaterThan(MAX_COMMAND_ARGS * MAX_COMMAND_ARG_BYTES); + }); + }); +}); + +function base(overrides: Partial<{ command: string[]; service: string; timeout: number; stdin: string }>) { + return { + command: overrides.command ?? ["ls"], + service: overrides.service ?? "web", + timeout: overrides.timeout ?? 60, + ...(overrides.stdin !== undefined ? { stdin: overrides.stdin } : {}) + }; +} diff --git a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts index ec3fd90949..ee400157bb 100644 --- a/apps/api/src/deployment/http-schemas/shell-exec.schema.ts +++ b/apps/api/src/deployment/http-schemas/shell-exec.schema.ts @@ -1,4 +1,38 @@ -import { z } from "zod"; +import { z } from "@hono/zod-openapi"; + +/** + * Request-side size limits for the shell-exec endpoint, exported as named + * constants so the schema, the route body limit, and the tests all derive from + * a single source of truth and cannot drift (per maintainer review on #3097). + * + * Real secret-injection payloads (env files, API keys, DB passwords, TLS + * cert/key chains, service-mesh tokens) are kilobytes — these caps are generous + * for the intended use while keeping per-request memory bounded so concurrent + * execs cannot be used as a cheap DoS. + */ +export const MAX_STDIN_BYTES = 16 * 1024; +export const MAX_COMMAND_ARGS = 64; +export const MAX_COMMAND_ARG_BYTES = 1024; +export const SERVICE_NAME_MAX = 253; + +/** + * Headroom for JSON key names, quotes, and escaping. The route body limit is + * measured on RAW request bytes (before JSON parsing), whereas the per-field + * byte caps above apply to the decoded strings (after parsing) — related but + * not identical. This overhead ensures a schema-valid request is never rejected + * by the body limit first. + */ +export const JSON_ENVELOPE_OVERHEAD_BYTES = 2 * 1024; + +/** + * Explicit request-body limit for the route, DERIVED arithmetically from the + * field caps above (never hardcoded) so it stays in sync automatically if any + * individual limit changes. + */ +export const SHELL_EXEC_BODY_LIMIT_BYTES = MAX_COMMAND_ARGS * MAX_COMMAND_ARG_BYTES + MAX_STDIN_BYTES + SERVICE_NAME_MAX + JSON_ENVELOPE_OVERHEAD_BYTES; + +/** UTF-8 byte length — the true wire size, unlike `.length`/`z.string().max()` which count UTF-16 code units. */ +const utf8Bytes = (value: string): number => Buffer.byteLength(value, "utf8"); export const ShellExecParamsSchema = z.object({ dseq: z.string().regex(/^\d+$/), @@ -7,16 +41,24 @@ export const ShellExecParamsSchema = z.object({ }); export const ShellExecRequestSchema = z.object({ - command: z.array(z.string().min(1)).min(1).max(64), - service: z.string().min(1).max(253), + command: z + .array( + z + .string() + .min(1) + .refine(s => utf8Bytes(s) <= MAX_COMMAND_ARG_BYTES, { message: `each command argument must not exceed ${MAX_COMMAND_ARG_BYTES} bytes (UTF-8)` }) + ) + .min(1) + .max(MAX_COMMAND_ARGS), + service: z.string().min(1).max(SERVICE_NAME_MAX), timeout: z.number().int().min(1).max(120).default(60), stdin: z .string() - .refine(s => Buffer.byteLength(s, "utf8") <= 1_048_576, { message: "stdin must not exceed 1 MiB (UTF-8 bytes)" }) + .refine(s => utf8Bytes(s) <= MAX_STDIN_BYTES, { message: `stdin must not exceed ${MAX_STDIN_BYTES} bytes (UTF-8)` }) .optional() .openapi({ description: - 'Optional raw UTF-8 data streamed to the command\'s standard input (max 1 MiB). Put secrets HERE (env files, tokens, passwords) — never in `command`, whose tokens are placed in the provider-proxy request URL, which is logged. Example command: ["sh","-c","cat > /run/secrets/.env"].', + 'Optional raw UTF-8 data streamed to the command\'s standard input (max 16 KiB). Put secrets HERE (env files, tokens, passwords) — never in `command`, whose tokens are placed in the provider-proxy request URL, which is logged. Example command: ["sh","-c","cat > /run/secrets/.env"].', example: "SECRET=value" }) }); diff --git a/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts index f7cb269edc..fa56a72648 100644 --- a/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts +++ b/apps/api/src/deployment/routes/shell-exec/shell-exec.router.ts @@ -4,7 +4,12 @@ import { createRoute } from "@src/core/lib/create-route/create-route"; import { OpenApiHonoHandler } from "@src/core/services/open-api-hono-handler/open-api-hono-handler"; import { SECURITY_BEARER_OR_API_KEY } from "@src/core/services/openapi-docs/openapi-security"; import { ShellExecController } from "@src/deployment/controllers/shell-exec/shell-exec.controller"; -import { ShellExecParamsSchema, ShellExecRequestSchema, ShellExecResponseSchema } from "@src/deployment/http-schemas/shell-exec.schema"; +import { + SHELL_EXEC_BODY_LIMIT_BYTES, + ShellExecParamsSchema, + ShellExecRequestSchema, + ShellExecResponseSchema +} from "@src/deployment/http-schemas/shell-exec.schema"; export const shellExecRouter = new OpenApiHonoHandler(); @@ -14,6 +19,9 @@ const shellExecRoute = createRoute({ summary: "Execute a shell command in a deployment container", tags: ["Shell Exec"], security: SECURITY_BEARER_OR_API_KEY, + // Explicit body limit derived from the schema field caps (see shell-exec.schema.ts) + // so it can never drift from them. Measured on raw request bytes (pre-parse). + bodyLimit: { maxSize: SHELL_EXEC_BODY_LIMIT_BYTES }, request: { params: ShellExecParamsSchema, body: { @@ -40,16 +48,22 @@ const shellExecRoute = createRoute({ description: "Unauthorized" }, 403: { - description: "Forbidden - user does not own this deployment" + description: "Forbidden - user does not own this deployment, or the provider authentication expired mid-execution" }, 404: { description: "Deployment or lease not found" }, + 413: { + description: "Request body exceeds the shell-exec body limit" + }, 500: { description: "Internal server error (e.g., lease provider address missing)" }, 502: { - description: "Provider proxy error" + description: "Provider proxy error (invalid provider host, connection failure, or provider-reported error)" + }, + 504: { + description: "Command execution timed out" } } }); diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts index cd85e3ee27..f3228ea14b 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.spec.ts @@ -9,6 +9,7 @@ import { buildShellUrl, isStrictBase64, isValidProviderHost, + MAX_OUTPUT_SIZE, parseExitCode, type ShellExecInput, type ShellExecOutput, @@ -199,7 +200,7 @@ describe(ShellExecService.name, () => { describe("execute - provider host pre-check", () => { it("returns an Err without opening a socket when the host is not https", async () => { - const { service } = createService(); + const { service } = setup(); const result = await service.execute(createShellExecInput({ providerBaseUrl: "http://provider.example.com" })); @@ -209,7 +210,7 @@ describe(ShellExecService.name, () => { }); it("returns an Err when the host is an IP address", async () => { - const { service } = createService(); + const { service } = setup(); const result = await service.execute(createShellExecInput({ providerBaseUrl: "https://203.0.113.10:8443" })); @@ -220,7 +221,7 @@ describe(ShellExecService.name, () => { describe("execute - proxy routing (D1)", () => { it("connects to the PROVIDER_PROXY_URL as wss and sends the envelope with the provider url and no data", async () => { - const { service, mockWs, getConstructorArgs } = createService(); + const { service, mockWs, getConstructorArgs } = setup(); const input = createShellExecInput(); const promise = service.execute(input); @@ -251,7 +252,7 @@ describe(ShellExecService.name, () => { const STDIN_MARKER = 104; it("sends a 104 stdin data frame plus a 104 EOF frame, keeping the secret out of the URL", async () => { - const { service, mockWs, getConstructorArgs } = createService(); + const { service, mockWs, getConstructorArgs } = setup(); const secret = `SECRET=${faker.string.alphanumeric(16)}\nAPI_KEY=${faker.string.alphanumeric(12)}`; const input = createShellExecInput({ command: ["sh", "-c", "cat > /run/secrets/.env && chmod 600 /run/secrets/.env"], @@ -306,7 +307,7 @@ describe(ShellExecService.name, () => { }); it("does not send any 104 frame when stdin is omitted", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const input = createShellExecInput(); const promise = service.execute(input); @@ -335,7 +336,7 @@ describe(ShellExecService.name, () => { describe("execute - receive / marker handling", () => { it("routes marker 100 to stdout with the marker byte stripped", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -350,7 +351,7 @@ describe(ShellExecService.name, () => { }); it("routes marker 101 to stderr with the marker byte stripped", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -365,7 +366,7 @@ describe(ShellExecService.name, () => { }); it("reads the exit code from a 102 JSON result frame", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -377,7 +378,7 @@ describe(ShellExecService.name, () => { }); it("reads the exit code from a 102 4-byte LE int32 result frame", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -389,7 +390,7 @@ describe(ShellExecService.name, () => { }); it("treats a 103 failure frame as a provider error (mapped 502), not output", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -402,7 +403,7 @@ describe(ShellExecService.name, () => { }); it("decodes a base64-string data payload", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); const base64 = Buffer.from([100, ...Buffer.from("hi", "utf-8")]).toString("base64"); @@ -418,7 +419,7 @@ describe(ShellExecService.name, () => { describe("execute - error / robustness handling", () => { it("resolves an error-key frame as a mapped provider error without decoding it", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -441,7 +442,7 @@ describe(ShellExecService.name, () => { }); it("drops a non-base64 string payload instead of leaking it as output", async () => { - const { service, mockWs, logger } = createService(); + const { service, mockWs, logger } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -456,7 +457,7 @@ describe(ShellExecService.name, () => { }); it("ignores pong keepalive frames without corrupting output", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -471,7 +472,7 @@ describe(ShellExecService.name, () => { }); it("returns an Err with the connection message on a socket error", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("error", new Error("ECONNREFUSED")); @@ -483,7 +484,7 @@ describe(ShellExecService.name, () => { }); it("returns an Err when the socket closes before an exit code arrives", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -496,12 +497,12 @@ describe(ShellExecService.name, () => { }); describe("execute - truncation (M5)", () => { - it("sets truncated but still reports the correct exit code when output exceeds 1 MB", async () => { - const { service, mockWs } = createService(); + it("sets truncated but still reports the correct exit code when output exceeds the cap", async () => { + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); - mockWs._trigger("message", dataFrame(100, "A".repeat(1024 * 1024 + 1))); + mockWs._trigger("message", dataFrame(100, "A".repeat(MAX_OUTPUT_SIZE + 1))); // keeps reading past the cap so the exit frame is still processed mockWs._trigger("message", exitFrameJson(3)); @@ -511,11 +512,11 @@ describe(ShellExecService.name, () => { expect((result.val as ShellExecOutput).exitCode).toBe(3); }); - it("does not truncate output that is exactly 1 MB", async () => { - const { service, mockWs } = createService(); + it("does not truncate output that is exactly at the cap", async () => { + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); - const data = "A".repeat(1024 * 1024); + const data = "A".repeat(MAX_OUTPUT_SIZE); mockWs._trigger("open"); mockWs._trigger("message", dataFrame(100, data)); mockWs._trigger("message", exitFrameJson(0)); @@ -523,13 +524,13 @@ describe(ShellExecService.name, () => { const result = await promise; expect(result.ok).toBe(true); expect((result.val as ShellExecOutput).truncated).toBe(false); - expect((result.val as ShellExecOutput).stdout.length).toBe(1024 * 1024); + expect((result.val as ShellExecOutput).stdout.length).toBe(MAX_OUTPUT_SIZE); }); }); describe("execute - auth expiry (M4)", () => { it("maps a 4001 close frame to an auth-expired error, not a generic provider error", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -542,7 +543,7 @@ describe(ShellExecService.name, () => { }); it("maps a 4003 ws close event to an auth-expired error", async () => { - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput()); mockWs._trigger("open"); @@ -557,7 +558,7 @@ describe(ShellExecService.name, () => { describe("execute - timeout", () => { it("resolves with a timeout error when the command runs past the timeout", async () => { vi.useFakeTimers(); - const { service, mockWs } = createService(); + const { service, mockWs } = setup(); const promise = service.execute(createShellExecInput({ timeout: 5 })); mockWs._trigger("open"); @@ -586,7 +587,7 @@ describe(ShellExecService.name, () => { }; } - function createService() { + function setup(overrides: { proxyUrl?: string } = {}) { const mockWs = createMockWebSocket(); let constructorArgs: unknown[] = []; vi.mocked(WebSocket).mockImplementation(function (this: unknown, ...args: unknown[]) { @@ -595,7 +596,7 @@ describe(ShellExecService.name, () => { } as unknown as typeof WebSocket); const logger = mock(); - const config = { PROVIDER_PROXY_URL: PROXY_URL } as DeploymentConfig; + const config = { PROVIDER_PROXY_URL: overrides.proxyUrl ?? PROXY_URL } as DeploymentConfig; const service = new ShellExecService(config, logger); return { service, mockWs, logger, getConstructorArgs: () => constructorArgs }; diff --git a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts index 062e0f6e87..53720f9855 100644 --- a/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts +++ b/apps/api/src/deployment/services/shell-exec/shell-exec.service.ts @@ -6,7 +6,14 @@ import { WebSocket } from "ws"; import { LoggerService } from "@src/core"; import { DEPLOYMENT_CONFIG, type DeploymentConfig } from "@src/deployment/config/config.provider"; -const MAX_OUTPUT_SIZE = 1024 * 1024; +/** + * Cap on the combined stdout+stderr the service buffers per exec, in UTF-8 + * bytes. Kept small (64 KiB) so many concurrent execs cannot exhaust memory — + * a synchronous buffered response is only safe while the buffer is bounded + * (per maintainer review on #3079/#3097). Exported so tests assert against the + * same constant instead of a hardcoded literal. + */ +export const MAX_OUTPUT_SIZE = 64 * 1024; /** * LeaseShell protocol markers. The provider prefixes every shell frame with a