AGE-1831: enhance DaytonaSandboxProvider with build failure handling

.changeset/20260817052904-daytona-snapshot-register-status.md

@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge-core': patch
+'@truefoundry/trueforge': patch
+---
+
+Await Daytona snapshot registration on sandbox provider configure so auth failures return 422 instead of a false pending status, and keep GET status refreshes persisted.

packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts

@@ -17,6 +17,8 @@ import { DEFAULT_PREVIEW_URL_EXPIRY_SECONDS, DEFAULT_SANDBOX_NATS_WS_PORT } from
 import type { ExecResult, SandboxBuild, SandboxExecParams, SandboxFileInfo, SandboxProvider } from './Provider';
 
 const SANDBOX_NOT_FOUND_STATUS = 404;
+/** Another replica already registered this build name; its create is the one that counts. */
+const SNAPSHOT_CONFLICT_STATUS = 409;
 const SANDBOX_STATE_STARTED = 'started';
 
 const BUILD_STATE_ACTIVE = 'active';
@@ -25,6 +27,12 @@ const BUILD_STATE_ERROR = 'error';
 const BUILD_STATE_BUILD_FAILED = 'build_failed';
 
 const IMAGE_BUILD_NAME_PREFIX = 'trueforge-build-';
+/** Same default the Daytona SDK applies when `DaytonaConfig.apiUrl` is omitted. */
+const DEFAULT_DAYTONA_API_URL = 'https://app.daytona.io/api';
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
 
 /**
  * Digest portion of a container image reference (the tag/digest after the final `:`)
@@ -60,6 +68,13 @@ function httpUrlToWsUrl(url: string): string {
 export interface DaytonaSandboxProviderOptions {
   /** Caller-owned Daytona SDK client (credentials / lifetime). */
   client: Daytona;
+  /**
+   * Same API key the client was built with; used for the register-only snapshot POST because the
+   * SDK's `snapshot.create` polls to a terminal state instead of returning once registered.
+   */
+  apiKey: string;
+  /** Daytona API base URL (including `/api`). Defaults to the SDK's public cloud endpoint. */
+  apiUrl?: string | undefined;
   tenantName: string;
   /** Release-owned sandbox image reference; built into a Daytona snapshot and cloned per sandbox. */
   sandboxImage: string;
@@ -94,6 +109,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
   private readonly fileMaxBytesForDownload: number;
   private readonly natsBridgePort: number;
   private readonly previewUrlExpirySeconds: number;
+  private readonly apiKey: string;
+  private readonly apiUrl: string;
   private readonly logger: Logger;
   private readonly daytona: Daytona;
   private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
@@ -102,6 +119,8 @@ export class DaytonaSandboxProvider implements SandboxProvider {
 
   constructor(options: DaytonaSandboxProviderOptions) {
     this.daytona = options.client;
+    this.apiKey = options.apiKey;
+    this.apiUrl = options.apiUrl ?? DEFAULT_DAYTONA_API_URL;
     this.tenantName = options.tenantName;
     this.imageUri = options.sandboxImage;
     this.buildRef = options.buildRef ?? deriveImageBuildName(imageDigest(options.sandboxImage));
@@ -243,7 +262,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
     }
   }
 
-  private toBuild(state: Snapshot['state'], errorReason: string | null): SandboxBuild {
+  private toBuild(state: string, errorReason: string | null): SandboxBuild {
     const metadata = { build_ref: this.buildRef, image_uri: this.imageUri };
     switch (state) {
       case BUILD_STATE_ACTIVE:
@@ -257,6 +276,45 @@ export class DaytonaSandboxProvider implements SandboxProvider {
     }
   }
 
+  /**
+   * Registers the snapshot and returns its initial state without waiting for the build.
+   *
+   * The SDK's `snapshot.create` issues this same POST and then polls until the snapshot is active
+   * or failed (minutes on a cold image pull). Configure only needs the registration result, so
+   * credential errors surface on the request while build progress stays observable via `getSnapshot`.
+   */
+  private async registerSnapshot(): Promise<{ state: string; errorReason: string | null }> {
+    let response: Response;
+    try {
+      response = await fetch(`${this.apiUrl}/snapshots`, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ name: this.buildRef, imageName: this.imageUri }),
+      });
+    } catch (error) {
+      throw new Error('Daytona snapshot registration request failed.', { cause: error });
+    }
+
+    const body: unknown = await response.json().catch(() => null);
+    if (!response.ok) {
+      const message =
+        isRecord(body) && typeof body['message'] === 'string'
+          ? body['message']
+          : `Daytona snapshot registration failed (${String(response.status)})`;
+      throw new DaytonaError(message, response.status);
+    }
+    if (!isRecord(body) || typeof body['state'] !== 'string') {
+      throw new DaytonaError("Failed to register snapshot. Daytona didn't return a snapshot state.");
+    }
+    return {
+      state: body['state'],
+      errorReason: typeof body['errorReason'] === 'string' ? body['errorReason'] : null,
+    };
+  }
+
   async buildImage(): Promise<SandboxBuild> {
     const existing = await this.getSnapshot(this.buildRef);
     if (existing) {
@@ -272,17 +330,24 @@ export class DaytonaSandboxProvider implements SandboxProvider {
       // re-saving settings (which calls buildImage) could never retry. Drop it, then recreate.
       await this.deleteFailedBuild(existing);
     }
-    // Fire the create in the background: the SDK's snapshot.create polls until the snapshot is
-    // terminal (minutes on a cold image pull), but Daytona registers it on the first request, so
-    // progress is observed via getImageBuildStatus. A concurrent-create conflict is harmless.
-    void this.daytona.snapshot.create({ name: this.buildRef, image: this.imageUri }).catch((error: unknown) => {
-      this.logger.error(`Daytona snapshot create failed: name=${this.buildRef}`, extractErrorLogFields(error));
-    });
-    return {
-      status: 'pending',
-      reason: 'Sandbox image build started.',
-      metadata: { build_ref: this.buildRef, image_uri: this.imageUri },
-    };
+
+    try {
+      const registered = await this.registerSnapshot();
+      // Registration returns pending almost always; map whatever Daytona sent so a fast
+      // active/error still surfaces correctly without a follow-up GET.
+      return this.toBuild(registered.state, registered.errorReason);
+    } catch (error) {
+      // A losing concurrent create is not a build failure: the winner owns the deterministic name.
+      if (error instanceof DaytonaError && error.statusCode === SNAPSHOT_CONFLICT_STATUS) {
+        this.logger.info(`Daytona snapshot already created concurrently: name=${this.buildRef}`);
+        return {
+          status: 'pending',
+          reason: 'Sandbox image build started by another server replica.',
+          metadata: { build_ref: this.buildRef, image_uri: this.imageUri },
+        };
+      }
+      throw error;
+    }
   }
 
   async getImageBuildStatus(): Promise<SandboxBuild> {

packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts

@@ -0,0 +1,80 @@
+import { Daytona, DaytonaError } from '@daytona/sdk';
+import { DaytonaSandboxProvider } from '../../../src/core/sandbox/provider/DaytonaProvider';
+import { makeSilentLogger } from '../harnessMocks';
+
+const NOT_FOUND_STATUS = 404;
+const CONFLICT_STATUS = 409;
+const FORBIDDEN_STATUS = 403;
+const API_URL = 'https://daytona.test/api';
+
+/**
+ * Builds a provider whose snapshot lookup reports "not built yet" so `buildImage` always reaches
+ * the register-only POST.
+ */
+function makeProvider(): DaytonaSandboxProvider {
+  // useDeprecatedPolling keeps the constructor from opening the event-stream WebSocket.
+  const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+  jest.spyOn(client.snapshot, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS));
+
+  return new DaytonaSandboxProvider({
+    client,
+    apiKey: 'dtn-test',
+    apiUrl: API_URL,
+    tenantName: 'test-tenant',
+    sandboxImage: 'registry.example.com/sandbox:029ea5ff',
+    timeoutMs: 1000,
+    autoStopIntervalInMinutes: 5,
+    autoArchiveIntervalInMinutes: 60,
+    autoDeleteIntervalInMinutes: 7200,
+    fileMaxBytesForDownload: 1024,
+    logger: makeSilentLogger(),
+  });
+}
+
+function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction<typeof globalThis.fetch> {
+  return jest
+    .spyOn(globalThis, 'fetch')
+    .mockResolvedValue(new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }));
+}
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe('DaytonaSandboxProvider register-only snapshot create', () => {
+  it('awaits the register POST and returns pending without polling to active', async () => {
+    const fetchMock = mockFetch({
+      status: 200,
+      body: { id: 'snap-1', name: 'trueforge-build-029ea5ff', state: 'pending', errorReason: null },
+    });
+
+    const build = await makeProvider().buildImage();
+
+    expect(build.status).toBe('pending');
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(fetchMock.mock.calls[0]?.[0]).toBe(`${API_URL}/snapshots`);
+    const init = fetchMock.mock.calls[0]?.[1];
+    expect(init?.method).toBe('POST');
+    expect(JSON.parse(String(init?.body))).toEqual({
+      name: 'trueforge-build-029ea5ff',
+      imageName: 'registry.example.com/sandbox:029ea5ff',
+    });
+  });
+
+  it('treats a concurrent-create conflict as pending, not a thrown failure', async () => {
+    mockFetch({ status: CONFLICT_STATUS, body: { statusCode: CONFLICT_STATUS, message: 'Conflict' } });
+
+    const build = await makeProvider().buildImage();
+
+    expect(build.status).toBe('pending');
+  });
+
+  it('throws on Access denied so PUT can map it to 422', async () => {
+    mockFetch({ status: FORBIDDEN_STATUS, body: { statusCode: FORBIDDEN_STATUS, message: 'Access denied' } });
+
+    await expect(makeProvider().buildImage()).rejects.toMatchObject({
+      message: 'Access denied',
+      statusCode: FORBIDDEN_STATUS,
+    });
+  });
+});

packages/trueforge/src/apis/sandboxProviders.ts

@@ -14,7 +14,7 @@ import type { PutSandboxProviderRequest, SandboxProviderManifest } from '../sche
 import { MissingStoredSecretError, resolveStoredSecretValue, toRedactedSecretValue } from '../utils/secretRedaction';
 import { TENANT_ID } from './sessions';
 
-/** Cap the Daytona build kickoff so a slow/unreachable provider can't hold the request (or DB txn) open. */
+/** Cap the Daytona register round-trip so a slow/unreachable provider can't hold the request (or DB txn) open. */
 const BUILD_REQUEST_TIMEOUT_MS = 3_000;
 
 export interface SandboxProvidersRouterDeps<TTransaction> {

packages/trueforge/src/sandbox/providerUtils.ts

@@ -3,7 +3,7 @@ import { Daytona, DaytonaError } from '@daytona/sdk';
 import { DaytonaSandboxProvider, SANDBOX_IMAGE_URI, type SandboxBuild } from '@truefoundry/trueforge-core/core';
 import type { Logger } from 'winston';
 import configuration from '../config';
-import type { ISandboxProviderStore } from '../db/sandboxProviderStore';
+import type { ISandboxProviderStore, SandboxProviderRecord } from '../db/sandboxProviderStore';
 import {
   toDaytonaSandboxProviderInput,
   type SandboxBuildMetadata,
@@ -38,6 +38,7 @@ export function toDaytonaSandboxProvider({
   const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest);
   return new DaytonaSandboxProvider({
     client: new Daytona({ apiKey }),
+    apiKey,
     ...settings,
     tenantName: tenant_id,
     sandboxImage: build_metadata?.['image_uri'] ?? SANDBOX_IMAGE_URI,
@@ -56,6 +57,14 @@ export function toSandboxStatus(build: SandboxBuild): SandboxStatus {
   };
 }
 
+function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus {
+  return {
+    status: record.status,
+    status_reason: record.status_reason,
+    build_metadata: record.build_metadata,
+  };
+}
+
 // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead.
 const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000;
 
@@ -73,11 +82,7 @@ export async function checkSnapshotStatus({
     return undefined;
   }
 
-  const persisted: SandboxStatus = {
-    status: record.status,
-    status_reason: record.status_reason,
-    build_metadata: record.build_metadata,
-  };
+  const persisted = sandboxStatusFromRecord(record);
 
   const readyIsFresh =
     record.status === 'ready' && Date.now() - Date.parse(record.updated_at) < READY_REVALIDATE_INTERVAL_MS;
@@ -99,6 +104,6 @@ export async function checkSnapshotStatus({
     build = await provider.getImageBuildStatus();
   }
   const next = toSandboxStatus(build);
-  await store.updateSandboxStatus({ tenant_id, ...next });
-  return next;
+  const updated = await store.updateSandboxStatus({ tenant_id, ...next });
+  return updated ? sandboxStatusFromRecord(updated) : next;
 }