From 51ff6d4e72add8f670b808a2e6ce6a4798e11a3b Mon Sep 17 00:00:00 2001 From: jonasnobile Date: Fri, 24 Apr 2026 16:43:02 +0200 Subject: [PATCH 1/2] fix: extend SetTimeout paused-state guard like Connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Manager.SetTimeout had the same state-blind ExpiresAt check that Manager.Connect had before 88f0245 — a paused sandbox past its original running-TTL returned ErrExpired, so any SDK code path that calls sandbox.setTimeout() on a paused sandbox before connect() hit 410. Same bug class, same fix: the running-TTL check only applies to running sandboxes. Paused sandboxes follow the pause-cycle reaper (FreezeDuration → demote, StoppedGCAfter → destroy). Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 9 +++++++++ internal/sandbox/manager.go | 10 +++++++--- internal/sandbox/manager_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f4c0b..5eb623b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ All notable changes to edvabe land here. Format roughly follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## Unreleased + +### Fixed + +- `SetTimeout` no longer returns `ErrExpired` (410) for a paused + sandbox whose original running-TTL has lapsed — same state-guard + the `Connect` path got in v0.1.1. Matters when SDK code calls + `sandbox.setTimeout()` on a paused sandbox ahead of `connect()`. + ## v0.1.1 — 2026-04-24 ### Fixed diff --git a/internal/sandbox/manager.go b/internal/sandbox/manager.go index 0896969..a3e803f 100644 --- a/internal/sandbox/manager.go +++ b/internal/sandbox/manager.go @@ -390,8 +390,12 @@ func (m *Manager) Destroy(ctx context.Context, id string) error { } // SetTimeout resets the sandbox TTL from the current clock. Returns -// ErrNotFound if the sandbox is unknown or ErrExpired if it already -// lapsed (typically meaning EnforceTimeouts hasn't reaped it yet). +// ErrNotFound if the sandbox is unknown or ErrExpired if a *running* +// sandbox's TTL already lapsed. Paused sandboxes are exempt — same +// reasoning as Manager.Connect: they live on the pause-cycle reaper +// (FreezeDuration → demote, StoppedGCAfter → destroy), not the +// running-TTL. Extending a paused sandbox's timeout is normal SDK +// flow (e.g. `sandbox.setTimeout(...)` ahead of `connect()`). func (m *Manager) SetTimeout(id string, timeout time.Duration) error { m.mu.Lock() defer m.mu.Unlock() @@ -400,7 +404,7 @@ func (m *Manager) SetTimeout(id string, timeout time.Duration) error { return ErrNotFound } now := m.clock.Now() - if !s.ExpiresAt.After(now) { + if s.State != StatePaused && !s.ExpiresAt.After(now) { return ErrExpired } s.ExpiresAt = now.Add(timeout) diff --git a/internal/sandbox/manager_test.go b/internal/sandbox/manager_test.go index b072e9b..060113e 100644 --- a/internal/sandbox/manager_test.go +++ b/internal/sandbox/manager_test.go @@ -305,6 +305,34 @@ func TestSetTimeoutOnExpired(t *testing.T) { } } +func TestSetTimeoutExtendsPausedSandboxPastOriginalTTL(t *testing.T) { + // Mirror of the Connect fix: SetTimeout on a paused sandbox whose + // running-TTL has lapsed must succeed, because the SDK reconnect + // flow can call setTimeout() before connect(), and paused + // sandboxes live on the pause-cycle reaper, not the running-TTL. + clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) + m, _ := newTestManagerWithRuntime(t, clk) + ctx := context.Background() + + s, err := m.Create(ctx, CreateOptions{Timeout: 30 * time.Second}) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := m.Pause(ctx, s.ID); err != nil { + t.Fatalf("Pause: %v", err) + } + clk.Advance(5 * time.Minute) + + if err := m.SetTimeout(s.ID, 10*time.Minute); err != nil { + t.Errorf("SetTimeout on paused+past-TTL = %v, want nil", err) + } + got, _ := m.Get(s.ID) + wantExpiry := clk.Now().Add(10 * time.Minute) + if !got.ExpiresAt.Equal(wantExpiry) { + t.Errorf("ExpiresAt after SetTimeout = %v, want %v", got.ExpiresAt, wantExpiry) + } +} + func TestConnectExtendsTimeout(t *testing.T) { clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) m, _ := newTestManager(t, clk) From c12a4b768f51d024f01b25a2c2bdc078d8f1bac8 Mon Sep 17 00:00:00 2001 From: jonasnobile Date: Fri, 24 Apr 2026 16:43:27 +0200 Subject: [PATCH 2/2] feat: rehydrate sandbox registry from Docker labels on startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit edvabe's sandbox registry is in-memory, so an edvabe restart previously left running/paused containers in Docker but invisible to SDK reconnect (404 or 410 depending on state). Manager.Rehydrate rebuilds the registry from Docker-label + container-state metadata on serve startup. Sandbox-level identity is persisted as immutable Docker labels at Create time: template id, envd/traffic tokens, on-timeout mode. Mutable fields (ExpiresAt, PausedAt) are reset to conservative defaults on rehydration — running sandboxes get now+DefaultTimeout (SDK typically re-extends shortly after), paused get a past ExpiresAt so only the pause-cycle reaper owns them. Containers created before this change lack the new labels; they still rehydrate state + metadata + resource limits, and TemplateID is recovered via an image-label fallback so they remain destroyable. The envd access token isn't recoverable for them, so SDK reconnect will fail envd auth — documented in the CHANGELOG as a migration note. Rehydrate runs once before the HTTP listener starts, is non-fatal on error, and logs skipped containers via slog. The runtime interface gains ListManaged + a ManagedContainer DTO so the sandbox package owns the label interpretation and the runtime stays dumb. LabelMetaPrefix is centralized in the runtime package so docker, noop, and the sandbox-label reader share one source of truth for the user-metadata prefix. Tests: unit round-trip through noop runtime, plus stopped-resume, pre-upgrade-container, and no-clobber edge cases; integration test against real Docker verifies ListManaged returns faithful labels, state, and agent endpoint. End-to-end verified against a live paused sandbox. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 20 ++ cmd/edvabe/main.go | 6 + docs/05-architecture.md | 40 ++++ internal/runtime/docker/create.go | 15 +- internal/runtime/docker/list.go | 140 +++++++++++ internal/runtime/docker/runtime.go | 4 - .../docker/runtime_integration_test.go | 50 +++- internal/runtime/noop/noop.go | 51 ++++ internal/runtime/runtime.go | 51 ++++ internal/sandbox/labels.go | 117 ++++++++++ internal/sandbox/manager.go | 63 +++++ internal/sandbox/manager_test.go | 219 ++++++++++++++++++ internal/template/builder/executor_test.go | 3 + 13 files changed, 769 insertions(+), 10 deletions(-) create mode 100644 internal/runtime/docker/list.go create mode 100644 internal/sandbox/labels.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eb623b..98e664f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,26 @@ All notable changes to edvabe land here. Format roughly follows the `Connect` path got in v0.1.1. Matters when SDK code calls `sandbox.setTimeout()` on a paused sandbox ahead of `connect()`. +### Added + +- **Sandbox registry rehydration on startup.** `edvabe serve` now + enumerates every `edvabe.managed=true` container on the Docker + daemon and rebuilds its in-memory sandbox registry from Docker + labels + container state. Paused sandboxes survive edvabe + restarts and SDK `reconnect(sandboxId)` works across them. New + sandbox-level labels (stamped at Create time): + - `edvabe.sandbox.template.id` + - `edvabe.sandbox.template.alias` + - `edvabe.sandbox.token.envd` + - `edvabe.sandbox.token.traffic` + - `edvabe.sandbox.ontimeout` + + Containers created before this release (missing the new labels) + still rehydrate with `State` / `PauseMode` / resource limits / + metadata intact, and remain destroyable. The envd access token + is not recoverable for them, so SDK reconnect against pre-upgrade + sandboxes will fail envd auth — destroy and recreate. + ## v0.1.1 — 2026-04-24 ### Fixed diff --git a/cmd/edvabe/main.go b/cmd/edvabe/main.go index bf74c85..6298940 100644 --- a/cmd/edvabe/main.go +++ b/cmd/edvabe/main.go @@ -177,6 +177,12 @@ func serveCmd(args []string) { os.Exit(1) } + if n, err := mgr.Rehydrate(context.Background(), sandbox.DefaultTimeout); err != nil { + slog.Warn("rehydrate failed; starting with empty sandbox registry", "err", err) + } else if n > 0 { + slog.Info("rehydrated sandboxes from existing containers", "count", n) + } + fileCache, err := filecache.New(fileCacheDir()) if err != nil { fmt.Fprintf(os.Stderr, "serve: init file cache: %v\n", err) diff --git a/docs/05-architecture.md b/docs/05-architecture.md index 6ee1c92..471ab40 100644 --- a/docs/05-architecture.md +++ b/docs/05-architecture.md @@ -365,6 +365,46 @@ Phase 5 if wanted. check `X-Access-Token`. It just forwards the header. envd verifies it against the token we told it about via `/init`. +## Sandbox registry persistence + +edvabe holds its sandbox registry in memory (`internal/sandbox/manager.go` +→ `sandboxes map[string]*Sandbox`). There is no SQLite, no JSON state +file, no Redis. The source of truth for "which sandboxes exist" is +the Docker daemon itself: every sandbox container is stamped with +labels that encode the identity fields a future edvabe process needs +to reconstruct the in-memory entry. + +On startup, `serve` calls `Manager.Rehydrate`, which asks the runtime +for every `edvabe.managed=true` container and rebuilds the registry +entry for each. Labels stamped at Create time: + +| Label | Purpose | +|---|---| +| `edvabe.managed=true` | filter key for `ListManaged` | +| `edvabe.sandbox.id` | registry key — same as container name | +| `edvabe.sandbox.template.id` | template the sandbox was built from | +| `edvabe.sandbox.template.alias` | optional alias (e.g. `code-interpreter-v1`) | +| `edvabe.sandbox.token.envd` | envd access token minted at Create | +| `edvabe.sandbox.token.traffic` | traffic token minted at Create | +| `edvabe.sandbox.ontimeout` | `kill` or `pause` | +| `edvabe.meta.` | user-supplied metadata, one per key | + +Fields derivable from `docker inspect` (CPU/memory caps, env vars, +container created-at, running/paused/exited state) are NOT stamped +as labels — reading them off the live container avoids drift. + +Mutable fields — `ExpiresAt`, `PausedAt` — cannot be persisted in +labels (Docker labels are immutable after Create) so they are reset +on rehydration: running sandboxes get `now + DefaultTimeout` (SDK +usually re-extends shortly after), paused sandboxes get a past +`ExpiresAt` (their lifecycle is ruled by `FreezeDuration` / +`StoppedGCAfter`, not the running-TTL check). + +Pre-rehydration containers (created before v0.1.2) still reconstruct +with state + metadata + resource limits, but their envd/traffic +tokens cannot be recovered. SDK reconnect against those sandboxes +fails envd auth — operator should destroy and recreate. + ## Sandbox container hardening (or lack thereof) Sandbox containers are created with `seccomp=unconfined` and diff --git a/internal/runtime/docker/create.go b/internal/runtime/docker/create.go index 7c6d268..67d7547 100644 --- a/internal/runtime/docker/create.go +++ b/internal/runtime/docker/create.go @@ -29,12 +29,17 @@ func (r *Runtime) Create(ctx context.Context, req runtime.CreateRequest) (*runti port = defaultAgentPort } - labels := map[string]string{ - LabelSandboxID: req.SandboxID, - LabelManaged: "true", - } + labels := map[string]string{} + // Caller-supplied labels go in first so the runtime's own keys win + // on collision — sandbox-id and managed=true are load-bearing for + // ListManaged/Rehydrate and must not be overwritten. + for k, v := range req.Labels { + labels[k] = v + } + labels[LabelSandboxID] = req.SandboxID + labels[LabelManaged] = "true" for k, v := range req.Metadata { - labels[LabelMetaPrefix+k] = v + labels[runtime.LabelMetaPrefix+k] = v } var mounts []mount.Mount diff --git a/internal/runtime/docker/list.go b/internal/runtime/docker/list.go new file mode 100644 index 0000000..fe9f02b --- /dev/null +++ b/internal/runtime/docker/list.go @@ -0,0 +1,140 @@ +package docker + +import ( + "context" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + + "github.com/contember/edvabe/internal/runtime" +) + +// ListManaged enumerates containers labeled edvabe.managed=true (including +// paused and stopped) and returns a normalized view. Individual +// inspect failures are swallowed — the caller is rehydrating on +// startup, one bad container shouldn't abort the whole sweep. +// Containers in transitional states (dead / removing / created but +// never started) are filtered out so the manager only sees entries it +// can act on. +func (r *Runtime) ListManaged(ctx context.Context) ([]runtime.ManagedContainer, error) { + filters := make(client.Filters).Add("label", LabelManaged+"=true") + result, err := r.cli.ContainerList(ctx, client.ContainerListOptions{ + All: true, + Filters: filters, + }) + if err != nil { + return nil, fmt.Errorf("docker runtime: list managed: %w", err) + } + + log := slog.Default().With("component", "docker.ListManaged") + out := make([]runtime.ManagedContainer, 0, len(result.Items)) + for _, summary := range result.Items { + sandboxID := summary.Labels[LabelSandboxID] + if sandboxID == "" { + log.Warn("skipping managed container without sandbox-id label", "container_id", summary.ID) + continue + } + + inspect, err := r.cli.ContainerInspect(ctx, summary.ID, client.ContainerInspectOptions{}) + if err != nil { + log.Warn("skipping managed container: inspect failed", "sandbox_id", sandboxID, "container_id", summary.ID, "err", err) + continue + } + mc, ok := r.buildManagedContainer(sandboxID, inspect.Container) + if !ok { + status := "" + if inspect.Container.State != nil { + status = string(inspect.Container.State.Status) + } + log.Warn("skipping managed container in transitional state", "sandbox_id", sandboxID, "state", status) + continue + } + out = append(out, mc) + } + return out, nil +} + +func (r *Runtime) buildManagedContainer(sandboxID string, c container.InspectResponse) (runtime.ManagedContainer, bool) { + state, ok := normalizeState(c.State) + if !ok { + return runtime.ManagedContainer{}, false + } + + var host string + if state != runtime.ContainerStateStopped { + if h, err := extractBridgeIP(c, r.network); err == nil { + host = h + } + } + + if host != "" { + r.mu.Lock() + r.endpoints[sandboxID] = endpoint{host: host, port: defaultAgentPort} + r.mu.Unlock() + } + + envs := parseEnvList(c.Config.Env) + createdAt, _ := time.Parse(time.RFC3339Nano, c.Created) + + cpu := 0 + mem := 0 + if c.HostConfig != nil { + if n := c.HostConfig.NanoCPUs; n > 0 { + cpu = int(n / 1_000_000_000) + } + if m := c.HostConfig.Memory; m > 0 { + mem = int(m / (1024 * 1024)) + } + } + + labels := make(map[string]string, len(c.Config.Labels)) + for k, v := range c.Config.Labels { + labels[k] = v + } + + return runtime.ManagedContainer{ + SandboxID: sandboxID, + ContainerID: c.ID, + Image: c.Config.Image, + Labels: labels, + EnvVars: envs, + State: state, + CPUCount: cpu, + MemoryMB: mem, + CreatedAt: createdAt, + AgentHost: host, + AgentPort: defaultAgentPort, + }, true +} + +func normalizeState(s *container.State) (runtime.ContainerState, bool) { + if s == nil { + return "", false + } + switch string(s.Status) { + case "running": + return runtime.ContainerStateRunning, true + case "paused": + return runtime.ContainerStatePaused, true + case "exited": + return runtime.ContainerStateStopped, true + default: + return "", false + } +} + +func parseEnvList(env []string) map[string]string { + out := make(map[string]string, len(env)) + for _, kv := range env { + i := strings.IndexByte(kv, '=') + if i < 0 { + continue + } + out[kv[:i]] = kv[i+1:] + } + return out +} diff --git a/internal/runtime/docker/runtime.go b/internal/runtime/docker/runtime.go index 18561bd..3fdba71 100644 --- a/internal/runtime/docker/runtime.go +++ b/internal/runtime/docker/runtime.go @@ -28,10 +28,6 @@ const ( // LabelManaged is a truthy marker so operators can filter edvabe's // containers apart from hand-launched ones. LabelManaged = "edvabe.managed" - // LabelMetaPrefix is prepended to keys in CreateRequest.Metadata so - // user metadata can't collide with edvabe's own labels. - LabelMetaPrefix = "edvabe.meta." - defaultAgentPort = 49983 ) diff --git a/internal/runtime/docker/runtime_integration_test.go b/internal/runtime/docker/runtime_integration_test.go index 77afb66..aa1db6c 100644 --- a/internal/runtime/docker/runtime_integration_test.go +++ b/internal/runtime/docker/runtime_integration_test.go @@ -98,7 +98,7 @@ func TestDockerRuntimeCreateInspectDestroy(t *testing.T) { if got := inspect.Container.Config.Labels[LabelSandboxID]; got != sid { t.Errorf("label %s = %q, want %q", LabelSandboxID, got, sid) } - if got := inspect.Container.Config.Labels[LabelMetaPrefix+"owner"]; got != "runtime-test" { + if got := inspect.Container.Config.Labels[runtime.LabelMetaPrefix+"owner"]; got != "runtime-test" { t.Errorf("metadata label missing: got %q", got) } @@ -256,6 +256,54 @@ func TestDockerRuntimeCreateRelaxesSeccompAndAppArmor(t *testing.T) { } } +func TestDockerRuntimeListManagedReturnsLabeledContainers(t *testing.T) { + r := newTestRuntime(t) + ctx := context.Background() + + sid := uniqueSandboxID(t) + t.Cleanup(func() { _ = r.Destroy(ctx, sid) }) + + if _, err := r.Create(ctx, runtime.CreateRequest{ + SandboxID: sid, + Image: testImage, + Metadata: map[string]string{"owner": "alice"}, + Labels: map[string]string{"edvabe.sandbox.template.id": "base"}, + }); err != nil { + t.Fatalf("Create: %v", err) + } + + managed, err := r.ListManaged(ctx) + if err != nil { + t.Fatalf("ListManaged: %v", err) + } + + var found *runtime.ManagedContainer + for i := range managed { + if managed[i].SandboxID == sid { + found = &managed[i] + break + } + } + if found == nil { + t.Fatalf("ListManaged did not return sandbox %q; got %d entries", sid, len(managed)) + } + if found.State != runtime.ContainerStateRunning { + t.Errorf("State = %q, want running", found.State) + } + if found.Labels["edvabe.sandbox.template.id"] != "base" { + t.Errorf("template-id label not recovered: %v", found.Labels) + } + if found.Labels["edvabe.meta.owner"] != "alice" { + t.Errorf("metadata label not recovered: %v", found.Labels) + } + if found.AgentHost == "" { + t.Errorf("AgentHost empty for running container") + } + if found.AgentPort != 49983 { + t.Errorf("AgentPort = %d, want 49983", found.AgentPort) + } +} + func TestDockerRuntimeCreateRequiresID(t *testing.T) { r := newTestRuntime(t) ctx := context.Background() diff --git a/internal/runtime/noop/noop.go b/internal/runtime/noop/noop.go index 1391de7..0265162 100644 --- a/internal/runtime/noop/noop.go +++ b/internal/runtime/noop/noop.go @@ -17,6 +17,7 @@ type entry struct { handle *runtime.SandboxHandle image string envs map[string]string + labels map[string]string startCmd string readyCmd string cpuCount int @@ -59,10 +60,23 @@ func (r *Runtime) Create(ctx context.Context, req runtime.CreateRequest) (*runti AgentPort: port, CreatedAt: time.Now(), } + // Mirror the docker runtime's label-stamping: caller labels plus + // edvabe.meta.* for user metadata. Keeps the noop runtime faithful + // so ListManaged-based tests round-trip the same data that a real + // restart would. + labels := copyStrMap(req.Labels) + if labels == nil { + labels = map[string]string{} + } + for k, v := range req.Metadata { + labels[runtime.LabelMetaPrefix+k] = v + } + r.sandboxes[req.SandboxID] = &entry{ handle: h, image: req.Image, envs: copyStrMap(req.EnvVars), + labels: labels, startCmd: req.StartCmd, readyCmd: req.ReadyCmd, cpuCount: req.CPUCount, @@ -208,6 +222,43 @@ func (r *Runtime) BuildImage(ctx context.Context, req runtime.BuildRequest) erro return nil } +// ListManaged returns every tracked sandbox as a ManagedContainer. +// Unit tests use this path to exercise Rehydrate logic without a real +// Docker daemon — seed entries with Create first, optionally +// transition them via Pause/Stop. +func (r *Runtime) ListManaged(ctx context.Context) ([]runtime.ManagedContainer, error) { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]runtime.ManagedContainer, 0, len(r.sandboxes)) + for id, e := range r.sandboxes { + state := runtime.ContainerStateRunning + switch { + case e.stopped: + state = runtime.ContainerStateStopped + case e.paused: + state = runtime.ContainerStatePaused + } + host := e.handle.AgentHost + if state == runtime.ContainerStateStopped { + host = "" + } + out = append(out, runtime.ManagedContainer{ + SandboxID: id, + ContainerID: e.handle.ContainerID, + Image: e.image, + Labels: copyStrMap(e.labels), + EnvVars: copyStrMap(e.envs), + State: state, + CPUCount: e.cpuCount, + MemoryMB: e.memoryMB, + CreatedAt: e.handle.CreatedAt, + AgentHost: host, + AgentPort: e.handle.AgentPort, + }) + } + return out, nil +} + func (r *Runtime) AgentEndpoint(sandboxID string) (host string, port int, err error) { r.mu.RLock() defer r.mu.RUnlock() diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 9189602..52a2df5 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -57,6 +57,50 @@ type Runtime interface { // AgentEndpoint tells the reverse proxy where this sandbox's agent // listens. Returns host and port reachable from the edvabe process. AgentEndpoint(sandboxID string) (host string, port int, err error) + + // ListManaged returns every edvabe-managed container currently on + // the host, including paused and stopped ones. Used on startup to + // rehydrate the sandbox registry after edvabe restarts. Containers + // in transitional states (dead, removing, created-but-not-started) + // are filtered out. + ListManaged(ctx context.Context) ([]ManagedContainer, error) +} + +// LabelMetaPrefix is the Docker-label prefix under which the runtime +// stamps caller-supplied CreateRequest.Metadata. Shared across +// runtimes (docker, noop) and the sandbox package's rehydration +// reader so the stamping and extraction stay in sync. +const LabelMetaPrefix = "edvabe.meta." + +// ContainerState is the normalized runtime state reported by ListManaged. +// Backends map their native states into this small set so the sandbox +// manager doesn't need to know Docker-specific strings. +type ContainerState string + +const ( + ContainerStateRunning ContainerState = "running" + ContainerStatePaused ContainerState = "paused" // docker pause — RAM held + ContainerStateStopped ContainerState = "stopped" // docker stop — RAM freed +) + +// ManagedContainer is the runtime's view of a labeled container found +// by ListManaged. All sandbox-specific interpretation of Labels/EnvVars +// lives in the sandbox package — the runtime is dumb about semantics. +type ManagedContainer struct { + SandboxID string + ContainerID string + Image string + Labels map[string]string + EnvVars map[string]string + State ContainerState + CPUCount int + MemoryMB int + CreatedAt time.Time + // AgentHost / AgentPort are populated for running and paused-frozen + // containers (IP is assigned). Stopped containers have AgentHost "" + // — the IP is re-resolved on Start. + AgentHost string + AgentPort int } // CreateRequest is the input to Runtime.Create. @@ -85,6 +129,13 @@ type CreateRequest struct { CPUCount int // MemoryMB caps the sandbox's RSS. Zero means unlimited. MemoryMB int + // Labels is an arbitrary map of Docker-labels the caller wants + // stamped on the container. Sandbox manager uses it to persist + // sandbox-level metadata (template id, tokens, on-timeout) so + // Rehydrate can reconstruct the in-memory registry after edvabe + // restarts. Runtime-owned keys (edvabe.managed, edvabe.sandbox.id, + // edvabe.meta.*) still win if they collide. + Labels map[string]string } // SandboxHandle is the runtime's view of a created sandbox. diff --git a/internal/sandbox/labels.go b/internal/sandbox/labels.go new file mode 100644 index 0000000..2cdcc1d --- /dev/null +++ b/internal/sandbox/labels.go @@ -0,0 +1,117 @@ +package sandbox + +import ( + "strings" + + "github.com/contember/edvabe/internal/runtime" +) + +// Docker labels edvabe stamps at Create time so Rehydrate can +// reconstruct the in-memory Sandbox after edvabe restarts. Only +// immutable facts belong here — Docker labels can't be modified after +// a container is created. Mutable state (PauseMode, PausedAt) is +// derived from the live container state instead. +const ( + LabelTemplateID = "edvabe.sandbox.template.id" + LabelTemplateAlias = "edvabe.sandbox.template.alias" + LabelTokenEnvd = "edvabe.sandbox.token.envd" + LabelTokenTraffic = "edvabe.sandbox.token.traffic" + LabelOnTimeout = "edvabe.sandbox.ontimeout" + + // labelImageTemplateID is the template-id label the template build + // system stamps on the image itself. Used as a fallback when a + // pre-rehydration sandbox (containers created before the + // sandbox-level labels shipped) is being reconstructed. + labelImageTemplateID = "edvabe.template.id" +) + +// buildSandboxLabels returns the label map passed to the runtime for +// a new sandbox. Only non-empty values are included so the container's +// label set stays minimal. +func buildSandboxLabels(s *Sandbox) map[string]string { + labels := map[string]string{ + LabelTemplateID: s.TemplateID, + LabelTokenEnvd: s.EnvdToken, + LabelTokenTraffic: s.TrafficToken, + } + if s.Alias != "" { + labels[LabelTemplateAlias] = s.Alias + } + if s.OnTimeout != "" { + labels[LabelOnTimeout] = string(s.OnTimeout) + } + return labels +} + +// sandboxFromManaged reconstructs a Sandbox from what the runtime +// reported. Fields that can't be recovered from labels or inspect +// (ExpiresAt, PausedAt) are left to the caller to default. +func sandboxFromManaged(mc runtime.ManagedContainer) *Sandbox { + state := StateRunning + var pauseMode PauseMode + switch mc.State { + case runtime.ContainerStatePaused: + state = StatePaused + pauseMode = PauseFrozen + case runtime.ContainerStateStopped: + state = StatePaused + pauseMode = PauseStopped + } + + templateID := mc.Labels[LabelTemplateID] + if templateID == "" { + templateID = mc.Labels[labelImageTemplateID] + } + + return &Sandbox{ + ID: mc.SandboxID, + TemplateID: templateID, + Alias: mc.Labels[LabelTemplateAlias], + ContainerID: mc.ContainerID, + AgentHost: mc.AgentHost, + AgentPort: mc.AgentPort, + EnvdToken: mc.Labels[LabelTokenEnvd], + TrafficToken: mc.Labels[LabelTokenTraffic], + State: state, + PauseMode: pauseMode, + OnTimeout: OnTimeoutMode(mc.Labels[LabelOnTimeout]), + Metadata: extractMetadataFromLabels(mc.Labels), + EnvVars: extractUserEnvVars(mc.EnvVars), + CreatedAt: mc.CreatedAt, + CPUCount: mc.CPUCount, + MemoryMB: mc.MemoryMB, + } +} + +// extractMetadataFromLabels pulls user metadata (edvabe.meta.* keys) +// back out of the label map — mirror of the MetaPrefix stamping the +// docker runtime does on Create. +func extractMetadataFromLabels(labels map[string]string) map[string]string { + out := map[string]string{} + for k, v := range labels { + if strings.HasPrefix(k, runtime.LabelMetaPrefix) { + out[strings.TrimPrefix(k, runtime.LabelMetaPrefix)] = v + } + } + if len(out) == 0 { + return nil + } + return out +} + +// extractUserEnvVars drops the edvabe-injected EDVABE_* bootstrap vars +// so the Sandbox's EnvVars map only reflects what the user actually +// asked for at Create time. +func extractUserEnvVars(env map[string]string) map[string]string { + out := map[string]string{} + for k, v := range env { + if strings.HasPrefix(k, "EDVABE_") { + continue + } + out[k] = v + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/sandbox/manager.go b/internal/sandbox/manager.go index a3e803f..2fb2080 100644 --- a/internal/sandbox/manager.go +++ b/internal/sandbox/manager.go @@ -199,6 +199,60 @@ func NewManager(opts Options) (*Manager, error) { // Domain is the host:port edvabe reports in Sandbox responses. func (m *Manager) Domain() string { return m.domain } +// Rehydrate repopulates the sandbox registry from containers already on +// the runtime. Called once on startup so paused / running sandboxes +// survive edvabe restarts. Sandbox-level metadata (template id, tokens, +// on-timeout) is recovered from Docker labels stamped at Create time; +// Mutable fields that aren't persisted (ExpiresAt, PausedAt) get +// defaults: ExpiresAt = now + defaultTimeout for running sandboxes +// (SDK typically re-extends via SetTimeout anyway), PausedAt = now for +// paused sandboxes (the reaper will then hold them for FreezeDuration +// before demoting, which is conservative but safe). +// +// Per-container failures are logged and skipped — one orphan must not +// prevent edvabe from starting. Containers labeled edvabe.managed=true +// but missing the sandbox-id label are left alone; the operator can +// reap them by hand. +func (m *Manager) Rehydrate(ctx context.Context, defaultTimeout time.Duration) (int, error) { + if defaultTimeout <= 0 { + defaultTimeout = DefaultTimeout + } + managed, err := m.rt.ListManaged(ctx) + if err != nil { + return 0, fmt.Errorf("sandbox: rehydrate: list managed: %w", err) + } + + now := m.clock.Now() + m.mu.Lock() + defer m.mu.Unlock() + n := 0 + for _, mc := range managed { + // Skip sandboxes already live in the registry — a later + // Rehydrate call (or a second one on a warmed Manager) must + // not clobber in-memory mutable state with stale labels. + if _, exists := m.sandboxes[mc.SandboxID]; exists { + continue + } + s := sandboxFromManaged(mc) + if s.State == StatePaused { + s.PausedAt = now + // Give paused sandboxes a nominal ExpiresAt in the past so + // the running-TTL reaper ignores them (only the pause-cycle + // reaper applies). Connect now skips ExpiresAt for paused + // sandboxes (see manager.Connect). + s.ExpiresAt = mc.CreatedAt + } else { + s.ExpiresAt = now.Add(defaultTimeout) + } + if s.OnTimeout == "" { + s.OnTimeout = OnTimeoutKill + } + m.sandboxes[mc.SandboxID] = s + n++ + } + return n, nil +} + // CreateOptions is the subset of the NewSandbox request body the Manager // cares about. The control-plane handler translates HTTP JSON into this. type CreateOptions struct { @@ -255,6 +309,14 @@ func (m *Manager) Create(ctx context.Context, opts CreateOptions) (*Sandbox, err envdToken := NewEnvdToken() trafficToken := NewTrafficToken() + labels := buildSandboxLabels(&Sandbox{ + TemplateID: templateID, + Alias: opts.Alias, + EnvdToken: envdToken, + TrafficToken: trafficToken, + OnTimeout: opts.OnTimeout, + }) + handle, err := m.rt.Create(ctx, runtime.CreateRequest{ SandboxID: id, Image: resolution.ImageTag, @@ -267,6 +329,7 @@ func (m *Manager) Create(ctx context.Context, opts CreateOptions) (*Sandbox, err ReadyCmd: resolution.ReadyCmd, CPUCount: cpuCount, MemoryMB: memoryMB, + Labels: labels, }) if err != nil { return nil, fmt.Errorf("sandbox: create %q: runtime: %w", id, err) diff --git a/internal/sandbox/manager_test.go b/internal/sandbox/manager_test.go index 060113e..42fc1eb 100644 --- a/internal/sandbox/manager_test.go +++ b/internal/sandbox/manager_test.go @@ -694,6 +694,225 @@ func TestConnectResumesPausedSandboxPastOriginalTTL(t *testing.T) { } } +func TestRehydrateReconstructsFromExistingContainers(t *testing.T) { + // Round-trip: Create a sandbox through a first manager, drop that + // manager, point a fresh one at the same runtime (simulating + // edvabe restarting while containers stay up on Docker), and call + // Rehydrate. The sandbox should reappear in the new registry with + // its identity labels recovered. + clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) + rt := noop.New() + m1, err := NewManager(Options{Runtime: rt, Agent: &stubAgent{port: 49983}, Clock: clk}) + if err != nil { + t.Fatalf("NewManager #1: %v", err) + } + ctx := context.Background() + + created, err := m1.Create(ctx, CreateOptions{ + TemplateID: "my-template", + Alias: "my-alias", + OnTimeout: OnTimeoutPause, + Metadata: map[string]string{"owner": "alice"}, + EnvVars: map[string]string{"FOO": "bar"}, + Timeout: time.Minute, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := m1.Pause(ctx, created.ID); err != nil { + t.Fatalf("Pause: %v", err) + } + + // Fresh manager, same runtime — empty in-memory registry. + clk.Advance(30 * time.Minute) + m2, err := NewManager(Options{Runtime: rt, Agent: &stubAgent{port: 49983}, Clock: clk}) + if err != nil { + t.Fatalf("NewManager #2: %v", err) + } + if _, err := m2.Get(created.ID); !errors.Is(err, ErrNotFound) { + t.Fatalf("fresh manager should not know about %q yet: err=%v", created.ID, err) + } + + n, err := m2.Rehydrate(ctx, 10*time.Minute) + if err != nil { + t.Fatalf("Rehydrate: %v", err) + } + if n != 1 { + t.Errorf("Rehydrate count = %d, want 1", n) + } + + got, err := m2.Get(created.ID) + if err != nil { + t.Fatalf("Get after Rehydrate: %v", err) + } + if got.TemplateID != "my-template" { + t.Errorf("TemplateID = %q, want my-template", got.TemplateID) + } + if got.Alias != "my-alias" { + t.Errorf("Alias = %q, want my-alias", got.Alias) + } + if got.EnvdToken != created.EnvdToken { + t.Errorf("EnvdToken = %q, want %q (same as before restart)", got.EnvdToken, created.EnvdToken) + } + if got.TrafficToken != created.TrafficToken { + t.Errorf("TrafficToken = %q, want %q", got.TrafficToken, created.TrafficToken) + } + if got.OnTimeout != OnTimeoutPause { + t.Errorf("OnTimeout = %q, want pause", got.OnTimeout) + } + if got.State != StatePaused || got.PauseMode != PauseFrozen { + t.Errorf("state = %q / pauseMode = %q, want paused/frozen", got.State, got.PauseMode) + } + if got.Metadata["owner"] != "alice" { + t.Errorf("metadata not recovered: %v", got.Metadata) + } + // Paused sandbox should be resumable even though its original TTL + // has lapsed (from the earlier fix; rehydration gives ExpiresAt in + // the past on purpose for paused sandboxes). + if _, err := m2.Connect(ctx, created.ID, time.Minute); err != nil { + t.Errorf("Connect on rehydrated paused sandbox: %v", err) + } +} + +func TestRehydrateResumesStoppedSandbox(t *testing.T) { + // Simulates a sandbox that was demoted to docker-stop by the + // reaper before edvabe restarted. Rehydration should reconstruct + // it as State=paused / PauseMode=stopped, and a subsequent Connect + // must take the resume-stopped path: rt.Start + re-resolve agent + // endpoint + InitAgent with the recovered envd token. + clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) + rt := noop.New() + ap1 := &stubAgent{port: 49983} + m1, err := NewManager(Options{Runtime: rt, Agent: ap1, Clock: clk}) + if err != nil { + t.Fatalf("NewManager #1: %v", err) + } + ctx := context.Background() + created, err := m1.Create(ctx, CreateOptions{Timeout: time.Minute}) + if err != nil { + t.Fatalf("Create: %v", err) + } + + // Reaper-equivalent: flip the runtime to stopped without going + // through the manager (which is the state the demote-to-stopped + // transition leaves behind on a real docker stop). + if err := rt.Stop(ctx, created.ID); err != nil { + t.Fatalf("rt.Stop: %v", err) + } + + clk.Advance(2 * time.Hour) + ap2 := &stubAgent{port: 49983} + m2, err := NewManager(Options{Runtime: rt, Agent: ap2, Clock: clk}) + if err != nil { + t.Fatalf("NewManager #2: %v", err) + } + n, err := m2.Rehydrate(ctx, 10*time.Minute) + if err != nil { + t.Fatalf("Rehydrate: %v", err) + } + if n != 1 { + t.Errorf("Rehydrate count = %d, want 1", n) + } + + got, _ := m2.Get(created.ID) + if got.State != StatePaused || got.PauseMode != PauseStopped { + t.Errorf("rehydrated state = %q / %q, want paused/stopped", got.State, got.PauseMode) + } + if got.AgentHost != "" { + t.Errorf("AgentHost for stopped = %q, want empty", got.AgentHost) + } + + resumed, err := m2.Connect(ctx, created.ID, time.Minute) + if err != nil { + t.Fatalf("Connect on stopped rehydrated sandbox: %v", err) + } + if resumed.State != StateRunning { + t.Errorf("state after Connect = %q, want running", resumed.State) + } + if resumed.AgentHost == "" { + t.Errorf("AgentHost still empty after Connect (rt.Start should have re-resolved)") + } + if resumed.EnvdToken != created.EnvdToken { + t.Errorf("EnvdToken after rehydrate+resume = %q, want %q", resumed.EnvdToken, created.EnvdToken) + } + if ap2.inits != 1 { + t.Errorf("agent.InitAgent called %d times on resume, want 1", ap2.inits) + } +} + +func TestRehydratePreUpgradeContainerHasEmptyTokens(t *testing.T) { + // Containers created before the sandbox-level labels shipped will + // rehydrate (Manager.Get works, State/Destroy work) but with empty + // tokens. This documents the known limitation so a future change + // doesn't accidentally start synthesizing fake tokens that would + // silently break envd auth. + clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) + rt := noop.New() + ctx := context.Background() + + // Simulate a pre-upgrade container by calling the runtime directly + // with only the two load-bearing labels (managed + sandbox-id, + // which older edvabe already stamped). + if _, err := rt.Create(ctx, runtime.CreateRequest{ + SandboxID: "isb_preupgrade", + Image: "old", + Labels: map[string]string{}, // no edvabe.sandbox.* labels + }); err != nil { + t.Fatalf("rt.Create: %v", err) + } + + m, err := NewManager(Options{Runtime: rt, Agent: &stubAgent{port: 49983}, Clock: clk}) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + if _, err := m.Rehydrate(ctx, time.Minute); err != nil { + t.Fatalf("Rehydrate: %v", err) + } + + got, err := m.Get("isb_preupgrade") + if err != nil { + t.Fatalf("Get after Rehydrate: %v", err) + } + if got.EnvdToken != "" { + t.Errorf("EnvdToken = %q, want empty for pre-upgrade container", got.EnvdToken) + } + if got.TrafficToken != "" { + t.Errorf("TrafficToken = %q, want empty for pre-upgrade container", got.TrafficToken) + } + // Destroy still works on pre-upgrade entries — critical for cleanup. + if err := m.Destroy(ctx, "isb_preupgrade"); err != nil { + t.Errorf("Destroy on pre-upgrade sandbox: %v", err) + } +} + +func TestRehydrateSkipsAlreadyKnownSandboxes(t *testing.T) { + // If a sandbox is already in the registry (e.g. a second Rehydrate + // call) we must not clobber it — the live in-memory entry is more + // authoritative than the reconstructed one. + clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) + m, _ := newTestManagerWithRuntime(t, clk) + ctx := context.Background() + + s, err := m.Create(ctx, CreateOptions{Timeout: time.Minute}) + if err != nil { + t.Fatalf("Create: %v", err) + } + originalExpiry := s.ExpiresAt + + clk.Advance(10 * time.Minute) + n, err := m.Rehydrate(ctx, 5*time.Minute) + if err != nil { + t.Fatalf("Rehydrate: %v", err) + } + if n != 0 { + t.Errorf("Rehydrate should skip known sandboxes, got n=%d", n) + } + got, _ := m.Get(s.ID) + if !got.ExpiresAt.Equal(originalExpiry) { + t.Errorf("ExpiresAt was clobbered by Rehydrate: got %v, want %v", got.ExpiresAt, originalExpiry) + } +} + func TestSnapshotCommitsRuntimeImage(t *testing.T) { clk := newFakeClock(time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC)) m, rt := newTestManagerWithRuntime(t, clk) diff --git a/internal/template/builder/executor_test.go b/internal/template/builder/executor_test.go index 036fa11..de858ec 100644 --- a/internal/template/builder/executor_test.go +++ b/internal/template/builder/executor_test.go @@ -47,6 +47,9 @@ func (f *fakeRuntime) Stats(context.Context, string) (*runtime.Stats, error) { func (f *fakeRuntime) AgentEndpoint(string) (string, int, error) { return "", 0, errors.New("not implemented") } +func (f *fakeRuntime) ListManaged(context.Context) ([]runtime.ManagedContainer, error) { + return nil, nil +} func (f *fakeRuntime) BuildImage(ctx context.Context, req runtime.BuildRequest) error { f.mu.Lock()