feat: rehydrate sandbox registry from Docker labels on startup

CHANGELOG.md

@@ -3,6 +3,35 @@
 All notable changes to edvabe land here. Format roughly follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## Unreleased
+
+### Fixed
+
+- `SetTimeout` no longer returns `ErrExpired` (410) for a paused
+  sandbox whose original running-TTL has lapsed — same state-guard
+  the `Connect` path got in v0.1.1. Matters when SDK code calls
+  `sandbox.setTimeout()` on a paused sandbox ahead of `connect()`.
+
+### Added
+
+- **Sandbox registry rehydration on startup.** `edvabe serve` now
+  enumerates every `edvabe.managed=true` container on the Docker
+  daemon and rebuilds its in-memory sandbox registry from Docker
+  labels + container state. Paused sandboxes survive edvabe
+  restarts and SDK `reconnect(sandboxId)` works across them. New
+  sandbox-level labels (stamped at Create time):
+  - `edvabe.sandbox.template.id`
+  - `edvabe.sandbox.template.alias`
+  - `edvabe.sandbox.token.envd`
+  - `edvabe.sandbox.token.traffic`
+  - `edvabe.sandbox.ontimeout`
+
+  Containers created before this release (missing the new labels)
+  still rehydrate with `State` / `PauseMode` / resource limits /
+  metadata intact, and remain destroyable. The envd access token
+  is not recoverable for them, so SDK reconnect against pre-upgrade
+  sandboxes will fail envd auth — destroy and recreate.
+
 ## v0.1.1 — 2026-04-24
 
 ### Fixed

cmd/edvabe/main.go

@@ -177,6 +177,12 @@ func serveCmd(args []string) {
 		os.Exit(1)
 	}
 
+	if n, err := mgr.Rehydrate(context.Background(), sandbox.DefaultTimeout); err != nil {
+		slog.Warn("rehydrate failed; starting with empty sandbox registry", "err", err)
+	} else if n > 0 {
+		slog.Info("rehydrated sandboxes from existing containers", "count", n)
+	}
+
 	fileCache, err := filecache.New(fileCacheDir())
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "serve: init file cache: %v\n", err)

docs/05-architecture.md

@@ -365,6 +365,46 @@ Phase 5 if wanted.
   check `X-Access-Token`. It just forwards the header. envd verifies it
   against the token we told it about via `/init`.
 
+## Sandbox registry persistence
+
+edvabe holds its sandbox registry in memory (`internal/sandbox/manager.go`
+→ `sandboxes map[string]*Sandbox`). There is no SQLite, no JSON state
+file, no Redis. The source of truth for "which sandboxes exist" is
+the Docker daemon itself: every sandbox container is stamped with
+labels that encode the identity fields a future edvabe process needs
+to reconstruct the in-memory entry.
+
+On startup, `serve` calls `Manager.Rehydrate`, which asks the runtime
+for every `edvabe.managed=true` container and rebuilds the registry
+entry for each. Labels stamped at Create time:
+
+| Label | Purpose |
+|---|---|
+| `edvabe.managed=true` | filter key for `ListManaged` |
+| `edvabe.sandbox.id` | registry key — same as container name |
+| `edvabe.sandbox.template.id` | template the sandbox was built from |
+| `edvabe.sandbox.template.alias` | optional alias (e.g. `code-interpreter-v1`) |
+| `edvabe.sandbox.token.envd` | envd access token minted at Create |
+| `edvabe.sandbox.token.traffic` | traffic token minted at Create |
+| `edvabe.sandbox.ontimeout` | `kill` or `pause` |
+| `edvabe.meta.<k>` | user-supplied metadata, one per key |
+
+Fields derivable from `docker inspect` (CPU/memory caps, env vars,
+container created-at, running/paused/exited state) are NOT stamped
+as labels — reading them off the live container avoids drift.
+
+Mutable fields — `ExpiresAt`, `PausedAt` — cannot be persisted in
+labels (Docker labels are immutable after Create) so they are reset
+on rehydration: running sandboxes get `now + DefaultTimeout` (SDK
+usually re-extends shortly after), paused sandboxes get a past
+`ExpiresAt` (their lifecycle is ruled by `FreezeDuration` /
+`StoppedGCAfter`, not the running-TTL check).
+
+Pre-rehydration containers (created before v0.1.2) still reconstruct
+with state + metadata + resource limits, but their envd/traffic
+tokens cannot be recovered. SDK reconnect against those sandboxes
+fails envd auth — operator should destroy and recreate.
+
 ## Sandbox container hardening (or lack thereof)
 
 Sandbox containers are created with `seccomp=unconfined` and

internal/runtime/docker/create.go

@@ -29,12 +29,17 @@ func (r *Runtime) Create(ctx context.Context, req runtime.CreateRequest) (*runti
 		port = defaultAgentPort
 	}
 
-	labels := map[string]string{
-		LabelSandboxID: req.SandboxID,
-		LabelManaged:   "true",
-	}
+	labels := map[string]string{}
+	// Caller-supplied labels go in first so the runtime's own keys win
+	// on collision — sandbox-id and managed=true are load-bearing for
+	// ListManaged/Rehydrate and must not be overwritten.
+	for k, v := range req.Labels {
+		labels[k] = v
+	}
+	labels[LabelSandboxID] = req.SandboxID
+	labels[LabelManaged] = "true"
 	for k, v := range req.Metadata {
-		labels[LabelMetaPrefix+k] = v
+		labels[runtime.LabelMetaPrefix+k] = v
 	}
 
 	var mounts []mount.Mount

internal/runtime/docker/list.go

@@ -0,0 +1,140 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/moby/moby/api/types/container"
+	"github.com/moby/moby/client"
+
+	"github.com/contember/edvabe/internal/runtime"
+)
+
+// ListManaged enumerates containers labeled edvabe.managed=true (including
+// paused and stopped) and returns a normalized view. Individual
+// inspect failures are swallowed — the caller is rehydrating on
+// startup, one bad container shouldn't abort the whole sweep.
+// Containers in transitional states (dead / removing / created but
+// never started) are filtered out so the manager only sees entries it
+// can act on.
+func (r *Runtime) ListManaged(ctx context.Context) ([]runtime.ManagedContainer, error) {
+	filters := make(client.Filters).Add("label", LabelManaged+"=true")
+	result, err := r.cli.ContainerList(ctx, client.ContainerListOptions{
+		All:     true,
+		Filters: filters,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("docker runtime: list managed: %w", err)
+	}
+
+	log := slog.Default().With("component", "docker.ListManaged")
+	out := make([]runtime.ManagedContainer, 0, len(result.Items))
+	for _, summary := range result.Items {
+		sandboxID := summary.Labels[LabelSandboxID]
+		if sandboxID == "" {
+			log.Warn("skipping managed container without sandbox-id label", "container_id", summary.ID)
+			continue
+		}
+
+		inspect, err := r.cli.ContainerInspect(ctx, summary.ID, client.ContainerInspectOptions{})
+		if err != nil {
+			log.Warn("skipping managed container: inspect failed", "sandbox_id", sandboxID, "container_id", summary.ID, "err", err)
+			continue
+		}
+		mc, ok := r.buildManagedContainer(sandboxID, inspect.Container)
+		if !ok {
+			status := ""
+			if inspect.Container.State != nil {
+				status = string(inspect.Container.State.Status)
+			}
+			log.Warn("skipping managed container in transitional state", "sandbox_id", sandboxID, "state", status)
+			continue
+		}
+		out = append(out, mc)
+	}
+	return out, nil
+}
+
+func (r *Runtime) buildManagedContainer(sandboxID string, c container.InspectResponse) (runtime.ManagedContainer, bool) {
+	state, ok := normalizeState(c.State)
+	if !ok {
+		return runtime.ManagedContainer{}, false
+	}
+
+	var host string
+	if state != runtime.ContainerStateStopped {
+		if h, err := extractBridgeIP(c, r.network); err == nil {
+			host = h
+		}
+	}
+
+	if host != "" {
+		r.mu.Lock()
+		r.endpoints[sandboxID] = endpoint{host: host, port: defaultAgentPort}
+		r.mu.Unlock()
+	}
+
+	envs := parseEnvList(c.Config.Env)
+	createdAt, _ := time.Parse(time.RFC3339Nano, c.Created)
+
+	cpu := 0
+	mem := 0
+	if c.HostConfig != nil {
+		if n := c.HostConfig.NanoCPUs; n > 0 {
+			cpu = int(n / 1_000_000_000)
+		}
+		if m := c.HostConfig.Memory; m > 0 {
+			mem = int(m / (1024 * 1024))
+		}
+	}
+
+	labels := make(map[string]string, len(c.Config.Labels))
+	for k, v := range c.Config.Labels {
+		labels[k] = v
+	}
+
+	return runtime.ManagedContainer{
+		SandboxID:   sandboxID,
+		ContainerID: c.ID,
+		Image:       c.Config.Image,
+		Labels:      labels,
+		EnvVars:     envs,
+		State:       state,
+		CPUCount:    cpu,
+		MemoryMB:    mem,
+		CreatedAt:   createdAt,
+		AgentHost:   host,
+		AgentPort:   defaultAgentPort,
+	}, true
+}
+
+func normalizeState(s *container.State) (runtime.ContainerState, bool) {
+	if s == nil {
+		return "", false
+	}
+	switch string(s.Status) {
+	case "running":
+		return runtime.ContainerStateRunning, true
+	case "paused":
+		return runtime.ContainerStatePaused, true
+	case "exited":
+		return runtime.ContainerStateStopped, true
+	default:
+		return "", false
+	}
+}
+
+func parseEnvList(env []string) map[string]string {
+	out := make(map[string]string, len(env))
+	for _, kv := range env {
+		i := strings.IndexByte(kv, '=')
+		if i < 0 {
+			continue
+		}
+		out[kv[:i]] = kv[i+1:]
+	}
+	return out
+}

internal/runtime/docker/runtime.go

@@ -28,10 +28,6 @@ const (
 	// LabelManaged is a truthy marker so operators can filter edvabe's
 	// containers apart from hand-launched ones.
 	LabelManaged = "edvabe.managed"
-	// LabelMetaPrefix is prepended to keys in CreateRequest.Metadata so
-	// user metadata can't collide with edvabe's own labels.
-	LabelMetaPrefix = "edvabe.meta."
-
 	defaultAgentPort = 49983
 )
 

internal/runtime/docker/runtime_integration_test.go

@@ -98,7 +98,7 @@ func TestDockerRuntimeCreateInspectDestroy(t *testing.T) {
 	if got := inspect.Container.Config.Labels[LabelSandboxID]; got != sid {
 		t.Errorf("label %s = %q, want %q", LabelSandboxID, got, sid)
 	}
-	if got := inspect.Container.Config.Labels[LabelMetaPrefix+"owner"]; got != "runtime-test" {
+	if got := inspect.Container.Config.Labels[runtime.LabelMetaPrefix+"owner"]; got != "runtime-test" {
 		t.Errorf("metadata label missing: got %q", got)
 	}
 
@@ -256,6 +256,54 @@ func TestDockerRuntimeCreateRelaxesSeccompAndAppArmor(t *testing.T) {
 	}
 }
 
+func TestDockerRuntimeListManagedReturnsLabeledContainers(t *testing.T) {
+	r := newTestRuntime(t)
+	ctx := context.Background()
+
+	sid := uniqueSandboxID(t)
+	t.Cleanup(func() { _ = r.Destroy(ctx, sid) })
+
+	if _, err := r.Create(ctx, runtime.CreateRequest{
+		SandboxID: sid,
+		Image:     testImage,
+		Metadata:  map[string]string{"owner": "alice"},
+		Labels:    map[string]string{"edvabe.sandbox.template.id": "base"},
+	}); err != nil {
+		t.Fatalf("Create: %v", err)
+	}
+
+	managed, err := r.ListManaged(ctx)
+	if err != nil {
+		t.Fatalf("ListManaged: %v", err)
+	}
+
+	var found *runtime.ManagedContainer
+	for i := range managed {
+		if managed[i].SandboxID == sid {
+			found = &managed[i]
+			break
+		}
+	}
+	if found == nil {
+		t.Fatalf("ListManaged did not return sandbox %q; got %d entries", sid, len(managed))
+	}
+	if found.State != runtime.ContainerStateRunning {
+		t.Errorf("State = %q, want running", found.State)
+	}
+	if found.Labels["edvabe.sandbox.template.id"] != "base" {
+		t.Errorf("template-id label not recovered: %v", found.Labels)
+	}
+	if found.Labels["edvabe.meta.owner"] != "alice" {
+		t.Errorf("metadata label not recovered: %v", found.Labels)
+	}
+	if found.AgentHost == "" {
+		t.Errorf("AgentHost empty for running container")
+	}
+	if found.AgentPort != 49983 {
+		t.Errorf("AgentPort = %d, want 49983", found.AgentPort)
+	}
+}
+
 func TestDockerRuntimeCreateRequiresID(t *testing.T) {
 	r := newTestRuntime(t)
 	ctx := context.Background()