From 17ea607e9349e914df5df80d68d2f79ed2698a46 Mon Sep 17 00:00:00 2001 From: Aloento <11802769+Aloento@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:09:26 +0200 Subject: [PATCH 1/3] Add health endpoints and graceful shutdown Add /healthz (liveness, no DB) and /readyz (readiness, DB ping with a 3s timeout) as top-level routes without auth middleware, so pods can be probed before scaling beyond one replica. Drive server shutdown from a fresh 15s timeout context instead of the already-cancelled signal context, log shutdown failures as errors, and close the database pool in App.Shutdown. --- cmd/main.go | 10 ++++-- internal/api/health.go | 36 +++++++++++++++++++++ internal/api/health_test.go | 63 +++++++++++++++++++++++++++++++++++++ internal/api/routes.go | 1 + internal/app/app.go | 6 ++-- internal/db/db.go | 4 +++ 6 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 internal/api/health.go create mode 100644 internal/api/health_test.go diff --git a/cmd/main.go b/cmd/main.go index bad46f2..df90c93 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,6 +7,7 @@ import ( "net/http" "os/signal" "syscall" + "time" "go.uber.org/zap" @@ -45,8 +46,13 @@ func main() { <-ctx.Done() s.Log.Info("shutdown app") - if err = s.Shutdown(ctx); err != nil { - logger.Fatal("app shutdown failed", zap.Error(err)) + // The signal context is already cancelled, so the shutdown needs its own + // deadline to drain in-flight requests. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + if err = s.Shutdown(shutdownCtx); err != nil { + logger.Error("app shutdown failed", zap.Error(err)) } ch.Shutdown() diff --git a/internal/api/health.go b/internal/api/health.go new file mode 100644 index 0000000..7506e43 --- /dev/null +++ b/internal/api/health.go @@ -0,0 +1,36 @@ +package api + +import ( + "context" + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +// readinessTimeout bounds the database check behind /readyz. +const readinessTimeout = 3 * time.Second + +func (a *API) initHealthRoutes() { + a.r.GET("/healthz", a.livenessHandler()) + a.r.GET("/readyz", a.readinessHandler()) +} + +func (a *API) livenessHandler() gin.HandlerFunc { + return func(c *gin.Context) { + c.Status(http.StatusOK) + } +} + +func (a *API) readinessHandler() gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), readinessTimeout) + defer cancel() + + if err := a.db.Ping(ctx); err != nil { + c.Status(http.StatusServiceUnavailable) + return + } + c.Status(http.StatusOK) + } +} diff --git a/internal/api/health_test.go b/internal/api/health_test.go new file mode 100644 index 0000000..74a0f3f --- /dev/null +++ b/internal/api/health_test.go @@ -0,0 +1,63 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/stackmon/otc-status-dashboard/internal/conf" + "github.com/stackmon/otc-status-dashboard/internal/db" +) + +const unreachableDBAddr = "127.0.0.1:1" + +func newHealthTestAPI(t *testing.T) *API { + t.Helper() + + database, err := db.New(&conf.Config{DB: "postgres://user:pass@" + unreachableDBAddr + "/test"}) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + r := gin.New() + a := &API{r: r, db: database, log: zaptest.NewLogger(t)} + a.initHealthRoutes() + return a +} + +func TestHealthz_ReturnsOKWithoutDatabase(t *testing.T) { + a := newHealthTestAPI(t) + + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + w := httptest.NewRecorder() + a.r.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestReadyz_ReturnsServiceUnavailableWhenDatabaseIsDown(t *testing.T) { + a := newHealthTestAPI(t) + + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + w := httptest.NewRecorder() + a.r.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) +} + +func TestHealthRoutesAreRegisteredWithoutAuth(t *testing.T) { + a := newHealthTestAPI(t) + + paths := map[string]bool{} + for _, route := range a.r.Routes() { + if route.Method == http.MethodGet { + paths[route.Path] = true + } + } + assert.True(t, paths["/healthz"], "/healthz must be a top-level GET route") + assert.True(t, paths["/readyz"], "/readyz must be a top-level GET route") +} diff --git a/internal/api/routes.go b/internal/api/routes.go index e313d70..2048032 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ const ( // returned so that a misconfigured deployment fails at boot instead of serving // 500s on the first request. func (a *API) InitRoutes(openAPISpecPath string) error { + a.initHealthRoutes() a.initV2Routes() a.initRSSRoutes() diff --git a/internal/app/app.go b/internal/app/app.go index a26ced9..f23831f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -160,6 +160,8 @@ func (a *App) Shutdown(ctx context.Context) error { a.Log.Error("metrics server shutdown", zap.Error(err)) } } - // TODO: add a proper shutdown for a database - return a.srv.Shutdown(ctx) + if err := a.srv.Shutdown(ctx); err != nil { + return err + } + return a.DB.Close() } diff --git a/internal/db/db.go b/internal/db/db.go index 3cd0e86..268cdf6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -59,6 +59,10 @@ func (db *DB) Close() error { return db.sql.Close() } +func (db *DB) Ping(ctx context.Context) error { + return db.sql.PingContext(ctx) +} + type IncidentsParams struct { Types []string Status *event.Status From f607fe9665f98bfc0f3366a013bd9911622a7ee6 Mon Sep 17 00:00:00 2001 From: Aloento <11802769+Aloento@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:10:52 +0200 Subject: [PATCH 2/3] Stop the checker before closing the database pool Check runs synchronously, so shutting the checker down first lets an in-flight scan finish before App.Shutdown closes the pool. --- cmd/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index df90c93..0cd3d77 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -46,6 +46,11 @@ func main() { <-ctx.Done() s.Log.Info("shutdown app") + // Stop the checker before the pool is closed: Check runs synchronously, so + // this waits for an in-flight scan to finish before App.Shutdown closes the + // database pool. + ch.Shutdown() + // The signal context is already cancelled, so the shutdown needs its own // deadline to drain in-flight requests. shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) @@ -55,7 +60,5 @@ func main() { logger.Error("app shutdown failed", zap.Error(err)) } - ch.Shutdown() - logger.Info("app exited") } From 22bef492020d27938355b92bae1e245f54e1f320 Mon Sep 17 00:00:00 2001 From: Aloento <11802769+Aloento@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:16:49 +0200 Subject: [PATCH 3/3] Name the shutdown timeout constant --- cmd/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 0cd3d77..6e87adb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,6 +16,9 @@ import ( "github.com/stackmon/otc-status-dashboard/internal/conf" ) +// shutdownTimeout bounds the in-flight request drain after SIGTERM. +const shutdownTimeout = 15 * time.Second + func main() { c, err := conf.LoadConf() if err != nil { @@ -53,7 +56,7 @@ func main() { // The signal context is already cancelled, so the shutdown needs its own // deadline to drain in-flight requests. - shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() if err = s.Shutdown(shutdownCtx); err != nil {