feat: idle-timeout keepalive model with auto-resume

.gitignore

@@ -33,3 +33,6 @@ test/e2e/python/.pytest_cache/
 # TypeScript E2E test artifacts
 test/e2e/ts/node_modules/
 test/e2e/ts/package-lock.json
+
+# Vendored dependencies
+vendor/

cmd/edvabe/main.go

@@ -164,13 +164,15 @@ func serveCmd(args []string) {
 		domain = fmt.Sprintf("localhost:%d", *port)
 	}
 	mgr, err := sandbox.NewManager(sandbox.Options{
-		Runtime:        rt,
-		Agent:          ap,
-		Domain:         domain,
-		Resolver:       template.NewSandboxResolver(templateStore),
-		FreezeDuration: *freezeDuration,
-		MaxFrozen:      *maxFrozen,
-		StoppedGCAfter: *stoppedGCAfter,
+		Runtime:           rt,
+		Agent:             ap,
+		Domain:            domain,
+		Resolver:          template.NewSandboxResolver(templateStore),
+		FreezeDuration:    *freezeDuration,
+		MaxFrozen:         *maxFrozen,
+		StoppedGCAfter:    *stoppedGCAfter,
+		KeepaliveEnabled:  os.Getenv("EDVABE_KEEPALIVE_ENABLED") != "0",
+		KeepaliveCoalesce: envDurationOr("EDVABE_KEEPALIVE_COALESCE", time.Second),
 	})
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "serve: init manager: %v\n", err)
@@ -224,13 +226,13 @@ func serveCmd(args []string) {
 		FileSigner: fileSigner,
 		PublicBase: os.Getenv("EDVABE_PUBLIC_BASE"),
 	})
-	proxyHandler := api.NewProxy(mgr, rt)
+	proxyHandler := api.NewProxy(mgr, rt, mgr)
 	dashboardHandler := dashboard.NewHandler(dashboard.HandlerOptions{
 		Manager:   mgr,
 		Runtime:   rt,
 		Templates: templateStore,
 	})
-	handler := api.NewRouter(controlHandler, proxyHandler, dashboardHandler)
+	handler := api.NewRouter(controlHandler, proxyHandler, dashboardHandler, mgr.MarkActivity)
 
 	if *dnsListen != "" {
 		answerStr := *dnsAnswer

internal/api/control/sandboxes.go

@@ -56,6 +56,7 @@ type sandboxResponse struct {
 	Metadata           map[string]string `json:"metadata,omitempty"`
 	StartedAt          time.Time         `json:"startedAt"`
 	EndAt              time.Time         `json:"endAt"`
+	LastActiveAt       time.Time         `json:"lastActiveAt,omitempty"`
 }
 
 type sandboxDetailResponse struct {
@@ -68,6 +69,7 @@ type sandboxDetailResponse struct {
 	Network             sandboxNetworkConfig `json:"network"`
 	AllowInternetAccess bool                 `json:"allowInternetAccess"`
 	EnvVars             map[string]string    `json:"envVars,omitempty"`
+	LastActiveAt        time.Time            `json:"lastActiveAt"`
 }
 
 type sandboxLifecycle struct {
@@ -549,7 +551,8 @@ func toSandboxResponse(manager sandboxManager, provider versionProvider, sbx *sa
 		Domain:             manager.Domain(),
 		Metadata:           sbx.Metadata,
 		StartedAt:          sbx.CreatedAt,
-		EndAt:              sbx.ExpiresAt,
+		EndAt:              sbx.ExpiresAt(),
+		LastActiveAt:       sbx.LastActiveAt,
 	}
 }
 
@@ -567,6 +570,7 @@ func toSandboxDetailResponse(manager sandboxManager, provider versionProvider, s
 		Network:             sandboxNetworkConfig{AllowPublicTraffic: true, AllowOut: []string{}, DenyOut: []string{}},
 		AllowInternetAccess: true,
 		EnvVars:             sbx.EnvVars,
+		LastActiveAt:        sbx.LastActiveAt,
 	}
 	// Fall back to runtime stats for sandboxes created before this
 	// field existed (CPUCount/MemoryMB == 0 means "unlimited" on the

internal/api/dashboard/handler.go

@@ -86,6 +86,7 @@ type overviewSandbox struct {
 	PausedAt       string            `json:"pausedAt,omitempty"`
 	CreatedAt      string            `json:"createdAt"`
 	ExpiresAt      string            `json:"expiresAt"`
+	LastActiveAt   string            `json:"lastActiveAt,omitempty"`
 	Metadata       map[string]string `json:"metadata,omitempty"`
 	CPUUsedPercent float64           `json:"cpuUsedPercent,omitempty"`
 	MemoryUsedMiB  int64             `json:"memoryUsedMiB,omitempty"`
@@ -136,14 +137,15 @@ func serveOverview(opts HandlerOptions, w http.ResponseWriter, r *http.Request)
 
 	for _, s := range list {
 		entry := overviewSandbox{
-			ID:         s.ID,
-			TemplateID: s.TemplateID,
-			Alias:      s.Alias,
-			State:      s.State,
-			PauseMode:  s.PauseMode,
-			CreatedAt:  s.CreatedAt.UTC().Format("2006-01-02T15:04:05Z"),
-			ExpiresAt:  s.ExpiresAt.UTC().Format("2006-01-02T15:04:05Z"),
-			Metadata:   s.Metadata,
+			ID:           s.ID,
+			TemplateID:   s.TemplateID,
+			Alias:        s.Alias,
+			State:        s.State,
+			PauseMode:    s.PauseMode,
+			CreatedAt:    s.CreatedAt.UTC().Format("2006-01-02T15:04:05Z"),
+			ExpiresAt:    s.ExpiresAt().UTC().Format("2006-01-02T15:04:05Z"),
+			LastActiveAt: s.LastActiveAt.UTC().Format("2006-01-02T15:04:05Z"),
+			Metadata:     s.Metadata,
 		}
 		if !s.PausedAt.IsZero() {
 			entry.PausedAt = s.PausedAt.UTC().Format("2006-01-02T15:04:05Z")

internal/api/dispatch.go

@@ -26,7 +26,11 @@ const (
 // the headers are populated so the proxy handler doesn't have to
 // re-parse the host. Requests with neither go to the control plane
 // unchanged.
-func NewRouter(control, proxy, dashboard http.Handler) http.Handler {
+//
+// onActivity, if non-nil, is called for every data-plane request
+// (before proxying) so the sandbox manager can stamp LastActiveAt
+// and reset the idle-TTL countdown.
+func NewRouter(control, proxy, dashboard http.Handler, onActivity func(string)) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if dashboard != nil && strings.HasPrefix(r.URL.Path, "/dashboard") {
 			dashboard.ServeHTTP(w, r)
@@ -47,6 +51,9 @@ func NewRouter(control, proxy, dashboard http.Handler) http.Handler {
 			control.ServeHTTP(w, r)
 			return
 		}
+		if onActivity != nil {
+			onActivity(id)
+		}
 		proxy.ServeHTTP(w, r)
 	})
 }

internal/api/dispatch_test.go

@@ -10,7 +10,7 @@ func TestRouterRoutesToControlWhenNoSandbox(t *testing.T) {
 	var hits struct{ control, proxy int }
 	ctrl := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { hits.control++ })
 	prox := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { hits.proxy++ })
-	h := NewRouter(ctrl, prox, nil)
+	h := NewRouter(ctrl, prox, nil, nil)
 
 	req := httptest.NewRequest("GET", "http://localhost:3000/health", nil)
 	h.ServeHTTP(httptest.NewRecorder(), req)
@@ -24,7 +24,7 @@ func TestRouterRoutesToProxyViaHeader(t *testing.T) {
 	var hits struct{ control, proxy int }
 	ctrl := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { hits.control++ })
 	prox := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { hits.proxy++ })
-	h := NewRouter(ctrl, prox, nil)
+	h := NewRouter(ctrl, prox, nil, nil)
 
 	req := httptest.NewRequest("POST", "http://localhost:3000/process.Process/Start", nil)
 	req.Header.Set(HeaderSandboxID, "isb_abc")
@@ -40,7 +40,7 @@ func TestRouterRoutesToProxyViaHost(t *testing.T) {
 	var seen *http.Request
 	ctrl := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { t.Error("should not hit control") })
 	prox := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { seen = r })
-	h := NewRouter(ctrl, prox, nil)
+	h := NewRouter(ctrl, prox, nil, nil)
 
 	req := httptest.NewRequest("GET", "http://49983-isb_host.localhost:3000/files?path=/etc", nil)
 	req.Host = "49983-isb_host.localhost:3000"
@@ -61,7 +61,7 @@ func TestRouterHeaderWinsOverHost(t *testing.T) {
 	var seen *http.Request
 	ctrl := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { t.Error("should not hit control") })
 	prox := http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { seen = r })
-	h := NewRouter(ctrl, prox, nil)
+	h := NewRouter(ctrl, prox, nil, nil)
 
 	req := httptest.NewRequest("GET", "http://49999-isb_host.localhost:3000/files", nil)
 	req.Host = "49999-isb_host.localhost:3000"

internal/api/proxy.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"strconv"
+	"time"
 
 	"github.com/contember/edvabe/internal/sandbox"
 )
@@ -20,6 +21,16 @@ type SandboxLookup interface {
 	Get(id string) (*sandbox.Sandbox, error)
 }
 
+// AutoResumer resumes a paused sandbox on demand. Satisfied by
+// sandbox.Manager.Connect — which unpauses the container, re-inits
+// the agent for stopped sandboxes, and resets the idle clock. When
+// non-nil, the proxy calls it before forwarding any request to a
+// paused sandbox so data-plane traffic self-heals without requiring
+// an explicit SDK /connect call.
+type AutoResumer interface {
+	Connect(ctx context.Context, id string, timeout time.Duration) (*sandbox.Sandbox, error)
+}
+
 // AgentResolver turns a sandbox ID into the host:port of its in-sandbox
 // agent. Satisfied by runtime.Runtime in production.
 type AgentResolver interface {
@@ -39,7 +50,7 @@ type AgentResolver interface {
 // own transport layer — we don't need to touch Connection/Upgrade/etc.
 // explicitly. The request body is never read by edvabe; it flows
 // through as a stream.
-func NewProxy(lookup SandboxLookup, resolver AgentResolver) http.Handler {
+func NewProxy(lookup SandboxLookup, resolver AgentResolver, resumer AutoResumer) http.Handler {
 	rp := &httputil.ReverseProxy{
 		// Negative flush interval = flush after every write. Required
 		// for Connect-RPC server streams and NDJSON responses.
@@ -61,7 +72,8 @@ func NewProxy(lookup SandboxLookup, resolver AgentResolver) http.Handler {
 			return
 		}
 
-		if _, err := lookup.Get(id); err != nil {
+		sb, err := lookup.Get(id)
+		if err != nil {
 			if errors.Is(err, sandbox.ErrNotFound) {
 				WriteError(w, http.StatusNotFound, fmt.Sprintf("sandbox %q not found", id))
 				return
@@ -70,6 +82,13 @@ func NewProxy(lookup SandboxLookup, resolver AgentResolver) http.Handler {
 			return
 		}
 
+		if sb.State == sandbox.StatePaused && resumer != nil {
+			if _, err := resumer.Connect(r.Context(), id, sb.Timeout); err != nil {
+				WriteError(w, http.StatusServiceUnavailable, fmt.Sprintf("auto-resume %q: %v", id, err))
+				return
+			}
+		}
+
 		host, port, err := resolver.AgentEndpoint(id)
 		if err != nil {
 			WriteError(w, http.StatusInternalServerError,

internal/api/proxy_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"bufio"
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -77,7 +78,7 @@ func TestProxyForwardsRequest(t *testing.T) {
 
 	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{"isb_a": {ID: "isb_a"}}}
 	rt := &fakeResolver{host: host, port: port}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("POST", front.URL+"/files?path=/etc/hostname", nil)
@@ -110,7 +111,7 @@ func TestProxyForwardsRequest(t *testing.T) {
 func TestProxyReturns404OnUnknownSandbox(t *testing.T) {
 	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{}}
 	rt := &fakeResolver{host: "127.0.0.1", port: 1}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("GET", front.URL+"/health", nil)
@@ -139,7 +140,7 @@ func TestProxyReturns404OnUnknownSandbox(t *testing.T) {
 func TestProxyReturns400WhenHeaderMissing(t *testing.T) {
 	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{}}
 	rt := &fakeResolver{host: "127.0.0.1", port: 1}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	resp, err := http.Get(front.URL + "/health")
@@ -158,7 +159,7 @@ func TestProxyReturns502WhenUpstreamDown(t *testing.T) {
 	// Point the resolver at a port nothing listens on (port 1 is the
 	// TCPMUX well-known port; effectively always closed on dev laptops).
 	rt := &fakeResolver{host: "127.0.0.1", port: 1}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("GET", front.URL+"/health", nil)
@@ -188,7 +189,7 @@ func TestProxyPortOverride(t *testing.T) {
 	// The resolver returns a different port (1) so the only way the
 	// request can reach our backend is via the header override.
 	rt := &fakeResolver{host: host, port: 1}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("GET", front.URL+"/health", nil)
@@ -220,7 +221,7 @@ func TestProxyNoPortHeaderUsesDefault(t *testing.T) {
 
 	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{"isb_d": {ID: "isb_d"}}}
 	rt := &fakeResolver{host: host, port: port}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("GET", front.URL+"/health", nil)
@@ -262,7 +263,7 @@ func TestProxyStreamsResponse(t *testing.T) {
 
 	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{"isb_s": {ID: "isb_s"}}}
 	rt := &fakeResolver{host: host, port: port}
-	front := httptest.NewServer(NewProxy(mgr, rt))
+	front := httptest.NewServer(NewProxy(mgr, rt, nil))
 	defer front.Close()
 
 	req, _ := http.NewRequest("GET", front.URL+"/stream", nil)
@@ -306,3 +307,83 @@ func TestProxyStreamsResponse(t *testing.T) {
 		t.Errorf("chunks too close together: t1=%v t2=%v gap=%v want>=%v", t1, t2, gap, chunkDelay-50*time.Millisecond)
 	}
 }
+
+// fakeResumer is an AutoResumer stub that records the call and
+// optionally flips the sandbox to running (simulating a successful
+// resume).
+type fakeResumer struct {
+	called   bool
+	calledID string
+}
+
+func (f *fakeResumer) Connect(_ context.Context, id string, _ time.Duration) (*sandbox.Sandbox, error) {
+	f.called = true
+	f.calledID = id
+	return nil, nil
+}
+
+func TestAutoResumePausedSandbox(t *testing.T) {
+	var gotPath string
+	host, port, closeBackend := newBackend(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		w.WriteHeader(200)
+		fmt.Fprintln(w, "ok")
+	}))
+	defer closeBackend()
+
+	sb := &sandbox.Sandbox{ID: "isb_paused", State: sandbox.StatePaused, Timeout: 5 * time.Minute}
+	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{"isb_paused": sb}}
+	rt := &fakeResolver{host: host, port: port}
+	resumer := &fakeResumer{}
+	front := httptest.NewServer(NewProxy(mgr, rt, resumer))
+	defer front.Close()
+
+	req, _ := http.NewRequest("GET", front.URL+"/files", nil)
+	req.Header.Set(HeaderSandboxID, "isb_paused")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("proxy request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if !resumer.called {
+		t.Error("AutoResumer.Connect was not called for paused sandbox")
+	}
+	if resumer.calledID != "isb_paused" {
+		t.Errorf("Connect called with id=%q, want isb_paused", resumer.calledID)
+	}
+	if resp.StatusCode != 200 {
+		t.Errorf("status = %d, want 200 (resume+proxy should succeed)", resp.StatusCode)
+	}
+	if gotPath != "/files" {
+		t.Errorf("backend path = %q, want /files", gotPath)
+	}
+}
+
+func TestProxyNoResumeForRunningSandbox(t *testing.T) {
+	host, port, closeBackend := newBackend(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+	}))
+	defer closeBackend()
+
+	sb := &sandbox.Sandbox{ID: "isb_running", State: sandbox.StateRunning, Timeout: 5 * time.Minute}
+	mgr := &fakeLookup{byID: map[string]*sandbox.Sandbox{"isb_running": sb}}
+	rt := &fakeResolver{host: host, port: port}
+	resumer := &fakeResumer{}
+	front := httptest.NewServer(NewProxy(mgr, rt, resumer))
+	defer front.Close()
+
+	req, _ := http.NewRequest("GET", front.URL+"/files", nil)
+	req.Header.Set(HeaderSandboxID, "isb_running")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("proxy request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resumer.called {
+		t.Error("AutoResumer.Connect should not be called for running sandbox")
+	}
+}