diff --git a/cmd/main.go b/cmd/main.go index bad46f2..6e87adb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,6 +7,7 @@ import ( "net/http" "os/signal" "syscall" + "time" "go.uber.org/zap" @@ -15,6 +16,9 @@ import ( "github.com/stackmon/otc-status-dashboard/internal/conf" ) +// shutdownTimeout bounds the in-flight request drain after SIGTERM. +const shutdownTimeout = 15 * time.Second + func main() { c, err := conf.LoadConf() if err != nil { @@ -45,11 +49,19 @@ func main() { <-ctx.Done() s.Log.Info("shutdown app") - if err = s.Shutdown(ctx); err != nil { - logger.Fatal("app shutdown failed", zap.Error(err)) - } - + // Stop the checker before the pool is closed: Check runs synchronously, so + // this waits for an in-flight scan to finish before App.Shutdown closes the + // database pool. ch.Shutdown() + // The signal context is already cancelled, so the shutdown needs its own + // deadline to drain in-flight requests. + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + + if err = s.Shutdown(shutdownCtx); err != nil { + logger.Error("app shutdown failed", zap.Error(err)) + } + logger.Info("app exited") } diff --git a/internal/api/health.go b/internal/api/health.go new file mode 100644 index 0000000..7506e43 --- /dev/null +++ b/internal/api/health.go @@ -0,0 +1,36 @@ +package api + +import ( + "context" + "net/http" + "time" + + "github.com/gin-gonic/gin" +) + +// readinessTimeout bounds the database check behind /readyz. +const readinessTimeout = 3 * time.Second + +func (a *API) initHealthRoutes() { + a.r.GET("/healthz", a.livenessHandler()) + a.r.GET("/readyz", a.readinessHandler()) +} + +func (a *API) livenessHandler() gin.HandlerFunc { + return func(c *gin.Context) { + c.Status(http.StatusOK) + } +} + +func (a *API) readinessHandler() gin.HandlerFunc { + return func(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), readinessTimeout) + defer cancel() + + if err := a.db.Ping(ctx); err != nil { + c.Status(http.StatusServiceUnavailable) + return + } + c.Status(http.StatusOK) + } +} diff --git a/internal/api/health_test.go b/internal/api/health_test.go new file mode 100644 index 0000000..74a0f3f --- /dev/null +++ b/internal/api/health_test.go @@ -0,0 +1,63 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/stackmon/otc-status-dashboard/internal/conf" + "github.com/stackmon/otc-status-dashboard/internal/db" +) + +const unreachableDBAddr = "127.0.0.1:1" + +func newHealthTestAPI(t *testing.T) *API { + t.Helper() + + database, err := db.New(&conf.Config{DB: "postgres://user:pass@" + unreachableDBAddr + "/test"}) + require.NoError(t, err) + t.Cleanup(func() { _ = database.Close() }) + + r := gin.New() + a := &API{r: r, db: database, log: zaptest.NewLogger(t)} + a.initHealthRoutes() + return a +} + +func TestHealthz_ReturnsOKWithoutDatabase(t *testing.T) { + a := newHealthTestAPI(t) + + req := httptest.NewRequest(http.MethodGet, "/healthz", nil) + w := httptest.NewRecorder() + a.r.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestReadyz_ReturnsServiceUnavailableWhenDatabaseIsDown(t *testing.T) { + a := newHealthTestAPI(t) + + req := httptest.NewRequest(http.MethodGet, "/readyz", nil) + w := httptest.NewRecorder() + a.r.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) +} + +func TestHealthRoutesAreRegisteredWithoutAuth(t *testing.T) { + a := newHealthTestAPI(t) + + paths := map[string]bool{} + for _, route := range a.r.Routes() { + if route.Method == http.MethodGet { + paths[route.Path] = true + } + } + assert.True(t, paths["/healthz"], "/healthz must be a top-level GET route") + assert.True(t, paths["/readyz"], "/readyz must be a top-level GET route") +} diff --git a/internal/api/routes.go b/internal/api/routes.go index e313d70..2048032 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ const ( // returned so that a misconfigured deployment fails at boot instead of serving // 500s on the first request. func (a *API) InitRoutes(openAPISpecPath string) error { + a.initHealthRoutes() a.initV2Routes() a.initRSSRoutes() diff --git a/internal/app/app.go b/internal/app/app.go index a26ced9..f23831f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -160,6 +160,8 @@ func (a *App) Shutdown(ctx context.Context) error { a.Log.Error("metrics server shutdown", zap.Error(err)) } } - // TODO: add a proper shutdown for a database - return a.srv.Shutdown(ctx) + if err := a.srv.Shutdown(ctx); err != nil { + return err + } + return a.DB.Close() } diff --git a/internal/db/db.go b/internal/db/db.go index 3cd0e86..268cdf6 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -59,6 +59,10 @@ func (db *DB) Close() error { return db.sql.Close() } +func (db *DB) Ping(ctx context.Context) error { + return db.sql.PingContext(ctx) +} + type IncidentsParams struct { Types []string Status *event.Status