From f4ac6735c46887cd1b42fb7c4f115f1dca8e90f7 Mon Sep 17 00:00:00 2001 From: mfwolffe Date: Wed, 2 Sep 2026 03:37:20 -0400 Subject: [PATCH 1/3] web: cancel background context on shutdown so pool close cannot hang --- internal/web/server.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/web/server.go b/internal/web/server.go index 7fd92079..a57adb45 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -51,6 +51,14 @@ type Options struct { // Run boots the web server and blocks until shutdown. func Run(ctx context.Context, opts Options) error { + // Everything started below (pagecache LISTEN, metrics observers, + // pprof) hangs off this context. It is canceled on shutdown so + // goroutines holding pool connections release them before the + // deferred pool.Close(), which otherwise blocks until systemd's + // stop timeout SIGKILLs the process (observed: 90 s of 502s per + // deploy, 2026-09-02 sitrep). + ctx, cancelAll := context.WithCancel(ctx) + defer cancelAll() cfg, err := config.Load(nil) if err != nil { return err @@ -475,8 +483,18 @@ func Run(ctx context.Context, opts Options) error { shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.Web.ShutdownTimeout) defer cancel() - if err := srv.Shutdown(shutdownCtx); err != nil { - return fmt.Errorf("shutdown: %w", err) + shutdownErr := srv.Shutdown(shutdownCtx) + if shutdownErr != nil { + // Drain window elapsed with connections still active (long + // polls, SSE). Close them so their handlers see a canceled + // request context and release whatever they hold. + logger.Warn("shutdown: drain timed out; closing remaining connections", "error", shutdownErr) + _ = srv.Close() + } + // Stop background goroutines before the deferred pool.Close(). + cancelAll() + if shutdownErr != nil { + return fmt.Errorf("shutdown: %w", shutdownErr) } return nil } From a898a2b795dcf2409eab24944a2d89efd0e7e3c1 Mon Sep 17 00:00:00 2001 From: mfwolffe Date: Wed, 2 Sep 2026 03:37:21 -0400 Subject: [PATCH 2/3] systemd: bound web stop timeout at 30s --- deploy/systemd/shithubd-web.service | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/systemd/shithubd-web.service b/deploy/systemd/shithubd-web.service index bd8e374e..bbf54744 100644 --- a/deploy/systemd/shithubd-web.service +++ b/deploy/systemd/shithubd-web.service @@ -16,6 +16,9 @@ ExecStartPre=/usr/local/bin/shithubd migrate up ExecStart=/usr/local/bin/shithubd web Restart=on-failure RestartSec=2 +# Bound the stop phase. Run() drains for web.shutdown_timeout (10s) +# and then force-closes; anything beyond this is a bug, not a drain. +TimeoutStopSec=30 LimitNOFILE=65535 # Memory ceiling. The 2026-09-02 availability sitrep From 2447d03548b6431fb2bc4b194898527255fff771 Mon Sep 17 00:00:00 2001 From: mfwolffe Date: Wed, 2 Sep 2026 03:37:51 -0400 Subject: [PATCH 3/3] pagecache: pin that Listen releases its connection on cancel --- internal/cache/pagecache/notify_test.go | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/internal/cache/pagecache/notify_test.go b/internal/cache/pagecache/notify_test.go index 7e8e90b5..4f299465 100644 --- a/internal/cache/pagecache/notify_test.go +++ b/internal/cache/pagecache/notify_test.go @@ -126,3 +126,48 @@ func TestListen_BadPayloadSwallowed(t *testing.T) { t.Fatal("listener stuck after bad payload — must keep processing") } } + +// TestListen_ReleasesConnOnCancel pins the shutdown contract: once the +// context is canceled, Listen must release its LISTEN connection so a +// following pool.Close() returns promptly. Before the 2026-09-02 fix the +// web server never canceled this context on SIGTERM, pool.Close() +// blocked on the held connection, and systemd SIGKILLed the process +// after 90 s of 502s on every deploy. +func TestListen_ReleasesConnOnCancel(t *testing.T) { + t.Parallel() + pool := dbtest.NewTestDB(t) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + pagecache.Listen(ctx, pool, func(int64, string) {}, slog.New(slog.NewTextHandler(io.Discard, nil))) + }() + + deadline := time.Now().Add(5 * time.Second) + for pool.Stat().AcquiredConns() == 0 { + if time.Now().After(deadline) { + cancel() + t.Fatal("listener never acquired a connection") + } + time.Sleep(10 * time.Millisecond) + } + + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Listen did not return after cancel") + } + + closed := make(chan struct{}) + go func() { + pool.Close() + close(closed) + }() + select { + case <-closed: + case <-time.After(5 * time.Second): + t.Fatal("pool.Close blocked after listener cancel") + } +}