Add health endpoints and graceful shutdown

cmd/main.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"go.uber.org/zap"
 
@@ -15,6 +16,9 @@ import (
 	"github.com/stackmon/otc-status-dashboard/internal/conf"
 )
 
+// shutdownTimeout bounds the in-flight request drain after SIGTERM.
+const shutdownTimeout = 15 * time.Second
+
 func main() {
 	c, err := conf.LoadConf()
 	if err != nil {
@@ -45,11 +49,19 @@ func main() {
 	<-ctx.Done()
 	s.Log.Info("shutdown app")
 
-	if err = s.Shutdown(ctx); err != nil {
-		logger.Fatal("app shutdown failed", zap.Error(err))
-	}
-
+	// Stop the checker before the pool is closed: Check runs synchronously, so
+	// this waits for an in-flight scan to finish before App.Shutdown closes the
+	// database pool.
 	ch.Shutdown()
 
+	// The signal context is already cancelled, so the shutdown needs its own
+	// deadline to drain in-flight requests.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+
+	if err = s.Shutdown(shutdownCtx); err != nil {
+		logger.Error("app shutdown failed", zap.Error(err))
+	}
+
 	logger.Info("app exited")
 }

internal/api/health.go

@@ -0,0 +1,36 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+// readinessTimeout bounds the database check behind /readyz.
+const readinessTimeout = 3 * time.Second
+
+func (a *API) initHealthRoutes() {
+	a.r.GET("/healthz", a.livenessHandler())
+	a.r.GET("/readyz", a.readinessHandler())
+}
+
+func (a *API) livenessHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Status(http.StatusOK)
+	}
+}
+
+func (a *API) readinessHandler() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		ctx, cancel := context.WithTimeout(c.Request.Context(), readinessTimeout)
+		defer cancel()
+
+		if err := a.db.Ping(ctx); err != nil {
+			c.Status(http.StatusServiceUnavailable)
+			return
+		}
+		c.Status(http.StatusOK)
+	}
+}

internal/api/health_test.go

@@ -0,0 +1,63 @@
+package api
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/stackmon/otc-status-dashboard/internal/conf"
+	"github.com/stackmon/otc-status-dashboard/internal/db"
+)
+
+const unreachableDBAddr = "127.0.0.1:1"
+
+func newHealthTestAPI(t *testing.T) *API {
+	t.Helper()
+
+	database, err := db.New(&conf.Config{DB: "postgres://user:pass@" + unreachableDBAddr + "/test"})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = database.Close() })
+
+	r := gin.New()
+	a := &API{r: r, db: database, log: zaptest.NewLogger(t)}
+	a.initHealthRoutes()
+	return a
+}
+
+func TestHealthz_ReturnsOKWithoutDatabase(t *testing.T) {
+	a := newHealthTestAPI(t)
+
+	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	w := httptest.NewRecorder()
+	a.r.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+}
+
+func TestReadyz_ReturnsServiceUnavailableWhenDatabaseIsDown(t *testing.T) {
+	a := newHealthTestAPI(t)
+
+	req := httptest.NewRequest(http.MethodGet, "/readyz", nil)
+	w := httptest.NewRecorder()
+	a.r.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusServiceUnavailable, w.Code)
+}
+
+func TestHealthRoutesAreRegisteredWithoutAuth(t *testing.T) {
+	a := newHealthTestAPI(t)
+
+	paths := map[string]bool{}
+	for _, route := range a.r.Routes() {
+		if route.Method == http.MethodGet {
+			paths[route.Path] = true
+		}
+	}
+	assert.True(t, paths["/healthz"], "/healthz must be a top-level GET route")
+	assert.True(t, paths["/readyz"], "/readyz must be a top-level GET route")
+}

internal/api/routes.go

@@ -16,6 +16,7 @@ const (
 // returned so that a misconfigured deployment fails at boot instead of serving
 // 500s on the first request.
 func (a *API) InitRoutes(openAPISpecPath string) error {
+	a.initHealthRoutes()
 	a.initV2Routes()
 	a.initRSSRoutes()
 

internal/app/app.go

@@ -160,6 +160,8 @@ func (a *App) Shutdown(ctx context.Context) error {
 			a.Log.Error("metrics server shutdown", zap.Error(err))
 		}
 	}
-	// TODO: add a proper shutdown for a database
-	return a.srv.Shutdown(ctx)
+	if err := a.srv.Shutdown(ctx); err != nil {
+		return err
+	}
+	return a.DB.Close()
 }

internal/db/db.go

@@ -59,6 +59,10 @@ func (db *DB) Close() error {
 	return db.sql.Close()
 }
 
+func (db *DB) Ping(ctx context.Context) error {
+	return db.sql.PingContext(ctx)
+}
+
 type IncidentsParams struct {
 	Types        []string
 	Status       *event.Status