Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions server/internal/app/openapi/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12285,6 +12285,14 @@ components:
type: object
enabled:
type: boolean
resolveOpenIncidents:
type: boolean
description: >-
Close the check's open incidents (outage and degraded) in the same
request, with resolution type `check_disabled`, so they do not stay
open on a check that no longer runs. Only valid together with
`enabled: false`; otherwise a `VALIDATION_ERROR`. Defaults to false:
incidents stay open.
period:
type: string
regions:
Expand Down
1 change: 1 addition & 0 deletions server/internal/app/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2068,6 +2068,7 @@ func (s *Server) SetupRoutes(ctx context.Context) {
// Deleting a check resolves its open incidents (the DB layer does that);
// this tells whoever was paged, once, that it closed (spec 2026-10-08-02).
checksService.SetCheckDeletedIncidentNotifier(incidentsService)
checksService.SetCheckDisableIncidentResolver(incidentsService)
// The MCP surface builds its OWN statuspages.Service (mcp.NewHandler runs
// far earlier in this function), and every NewService starts with its own
// view memo. Point it at this one's, or an MCP-driven page edit evicts a map
Expand Down
4 changes: 4 additions & 0 deletions server/internal/db/models/incident.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ const (
// org deletion, the test API. The DB layer's DeleteCheck sets it in the
// same transaction as the soft delete, so no incident outlives its check.
ResolutionTypeCheckDeleted = "check_deleted"
// ResolutionTypeCheckDisabled closes an active incident because its check
// was disabled by hand with resolveOpenIncidents (spec 2026-10-23-01). The
// check did not necessarily recover: it simply stopped running.
ResolutionTypeCheckDisabled = "check_disabled"
)

// Incident kinds. `kind` discriminates what an incident row is ABOUT. It is a
Expand Down
14 changes: 7 additions & 7 deletions server/internal/email/templates/incident-resolved.html
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
{{define "subject"}}{{if .SubjectPrefix}}{{.SubjectPrefix}} {{end}}{{if .CheckDeleted}}⚪ {{t "incident.resolved.deletedSubject" "check" .CheckName}}{{else}}🟢 {{if .Duration}}{{t "incident.resolved.titleAfter" "check" .CheckName "duration" (duration .Duration)}}{{else}}{{t "incident.resolved.title" "check" .CheckName}}{{end}}{{end}}{{if .IncidentNumber}} (#{{.IncidentNumber}}){{end}}{{end}}
{{define "preheader"}}{{if .CheckDeleted}}{{t "incident.resolved.deletedPreheader" "check" .CheckName}}{{else}}{{if .Duration}}{{t "incident.resolved.downFor" "duration" (duration .Duration)}} {{end}}{{t "incident.resolved.passing"}}{{end}}{{end}}
{{define "subject"}}{{if .SubjectPrefix}}{{.SubjectPrefix}} {{end}}{{if .CheckDeleted}}⚪ {{t (print "incident.resolved." .ClosedKind "Subject") "check" .CheckName}}{{else}}🟢 {{if .Duration}}{{t "incident.resolved.titleAfter" "check" .CheckName "duration" (duration .Duration)}}{{else}}{{t "incident.resolved.title" "check" .CheckName}}{{end}}{{end}}{{if .IncidentNumber}} (#{{.IncidentNumber}}){{end}}{{end}}
{{define "preheader"}}{{if .CheckDeleted}}{{t (print "incident.resolved." .ClosedKind "Preheader") "check" .CheckName}}{{else}}{{if .Duration}}{{t "incident.resolved.downFor" "duration" (duration .Duration)}} {{end}}{{t "incident.resolved.passing"}}{{end}}{{end}}
{{template "base.html" .}}
{{define "statusbanner"}}{{if .CheckDeleted}}{{template "statusbanner-render" dict "Variant" "comment" "Label" (t "incident.pill.checkDeleted")}}{{else}}{{template "statusbanner-render" dict "Variant" "recovered" "Label" (t "incident.pill.recovered")}}{{end}}{{end}}
{{define "statusbanner"}}{{if .CheckDeleted}}{{template "statusbanner-render" dict "Variant" "comment" "Label" (t (print "incident.pill.check" .ClosedPill))}}{{else}}{{template "statusbanner-render" dict "Variant" "recovered" "Label" (t "incident.pill.recovered")}}{{end}}{{end}}
{{define "content"}}
{{$check := nolink .CheckName}}{{if .CheckURL}}{{$check = tHTML "incident.checkLink" "url" .CheckURL "check" $check}}{{end}}
<h1>{{if .CheckDeleted}}{{tHTML "incident.resolved.deletedTitle" "check" $check}}{{else}}{{tHTML "incident.resolved.title" "check" $check}}{{end}}</h1>
<h1>{{if .CheckDeleted}}{{tHTML (print "incident.resolved." .ClosedKind "Title") "check" $check}}{{else}}{{tHTML "incident.resolved.title" "check" $check}}{{end}}</h1>
{{if .CheckDeleted}}
<p>{{t "incident.resolved.deletedBody"}}</p>
<p>{{t (print "incident.resolved." .ClosedKind "Body")}}</p>
{{else if .Duration}}
<p class="lead">{{tHTML "incident.resolved.leadDuration" "duration" (duration .Duration)}}</p>
{{else}}
Expand All @@ -23,9 +23,9 @@ <h1>{{if .CheckDeleted}}{{tHTML "incident.resolved.deletedTitle" "check" $check}
<div class="cta-wrap">{{template "button" dict "URL" .IncidentURL "Label" (t "common.incidentActions.viewIncident") "Variant" "secondary"}}</div>
{{end}}
{{end}}
{{define "text"}}{{if .CheckDeleted}}{{t "incident.resolved.deletedTitle" "check" .CheckName}}
{{define "text"}}{{if .CheckDeleted}}{{t (print "incident.resolved." .ClosedKind "Title") "check" .CheckName}}

{{t "incident.resolved.deletedBody"}}
{{t (print "incident.resolved." .ClosedKind "Body")}}
{{else}}{{t "incident.resolved.textTitle" "check" .CheckName}}

{{t "incident.resolved.textBody" "check" .CheckName}}
Expand Down
121 changes: 121 additions & 0 deletions server/internal/handlers/checks/check_disable_incidents_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package checks_test

// Disabling a check with resolveOpenIncidents closes its open incidents with
// resolution type check_disabled (spec 2026-10-23-01).

import (
"encoding/json"
"net/http"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/fclairamb/solidping/server/internal/db/models"
"github.com/fclairamb/solidping/server/internal/handlers/checks"
)

func TestDisableResolvesOpenIncidents(t *testing.T) {
t.Parallel()
r := require.New(t)
ctx := t.Context()
world := newDegradedDisableWorld(t, "disable-resolves")
world.checks.SetCheckDisableIncidentResolver(world.incidents)

check := models.NewCheck(world.org.UID, "acme", "http")
r.NoError(world.dbSvc.CreateCheck(ctx, check))

outage := models.NewIncident(world.org.UID, check.UID, time.Now().Add(-time.Hour), "acme is down")
r.NoError(world.dbSvc.CreateIncident(ctx, outage))
degraded := world.openDegraded(t, check.UID)

// Positive control: disabling without the flag leaves both active.
_, err := world.checks.UpdateCheck(ctx, world.org.Slug, check.UID, &checks.UpdateCheckRequest{
Enabled: boolPtr(false),
})
r.NoError(err)
r.Equal(models.IncidentStateActive, world.incident(t, outage.UID).State)
r.Equal(models.IncidentStateActive, world.incident(t, degraded.UID).State)

_, err = world.checks.UpdateCheck(ctx, world.org.Slug, check.UID, &checks.UpdateCheckRequest{
Enabled: boolPtr(false),
ResolveOpenIncidents: boolPtr(true),
})
r.NoError(err)

for _, uid := range []string{outage.UID, degraded.UID} {
got := world.incident(t, uid)
r.Equal(models.IncidentStateResolved, got.State)
r.NotNil(got.ResolutionType)
r.Equal(models.ResolutionTypeCheckDisabled, *got.ResolutionType)
r.NotNil(got.ResolvedAt)

events, listErr := world.dbSvc.ListEvents(ctx, &models.ListEventsFilter{
OrganizationUID: world.org.UID,
IncidentUID: &uid,
EventTypes: []models.EventType{models.EventTypeIncidentResolved},
})
r.NoError(listErr)
r.Len(events, 1, "one incident.resolved event per incident")
}

// A retry with nothing open is a no-op, and emits no second event.
_, err = world.checks.UpdateCheck(ctx, world.org.Slug, check.UID, &checks.UpdateCheckRequest{
Enabled: boolPtr(false),
ResolveOpenIncidents: boolPtr(true),
})
r.NoError(err)

for _, uid := range []string{outage.UID, degraded.UID} {
events, listErr := world.dbSvc.ListEvents(ctx, &models.ListEventsFilter{
OrganizationUID: world.org.UID,
IncidentUID: &uid,
EventTypes: []models.EventType{models.EventTypeIncidentResolved},
})
r.NoError(listErr)
r.Len(events, 1, "retry must not emit a second incident.resolved event")
}
}

func TestResolveOpenIncidentsWithoutDisableOverHTTPIs400(t *testing.T) {
t.Parallel()
r := require.New(t)

router := newFullChecksRouter(t)
base := "/orgs/acme/checks"

body := `{"type":"http","name":"acme","config":{"url":"https://acme.com"}}`
rec := checksCall(t, router, http.MethodPost, base, body)
r.Equal(http.StatusCreated, rec.Code, rec.Body.String())
var created struct {
UID string `json:"uid"`
}
r.NoError(json.Unmarshal(rec.Body.Bytes(), &created))

rec = checksCall(t, router, http.MethodPatch, base+"/"+created.UID, `{"resolveOpenIncidents":true}`)
r.Equal(http.StatusBadRequest, rec.Code, rec.Body.String())
r.Contains(rec.Body.String(), "VALIDATION_ERROR")
}

func TestResolveOpenIncidentsWithoutDisableIsRejected(t *testing.T) {
t.Parallel()
r := require.New(t)
ctx := t.Context()
world := newDegradedDisableWorld(t, "disable-rejects")
world.checks.SetCheckDisableIncidentResolver(world.incidents)

check := models.NewCheck(world.org.UID, "acme", "http")
r.NoError(world.dbSvc.CreateCheck(ctx, check))
outage := models.NewIncident(world.org.UID, check.UID, time.Now().Add(-time.Hour), "acme is down")
r.NoError(world.dbSvc.CreateIncident(ctx, outage))

for _, req := range []*checks.UpdateCheckRequest{
{ResolveOpenIncidents: boolPtr(true)},
{Enabled: boolPtr(true), ResolveOpenIncidents: boolPtr(true)},
} {
_, err := world.checks.UpdateCheck(ctx, world.org.Slug, check.UID, req)
r.Error(err)
}

r.Equal(models.IncidentStateActive, world.incident(t, outage.UID).State)
}
54 changes: 54 additions & 0 deletions server/internal/handlers/checks/disable_incidents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package checks

import (
"context"
"errors"
"fmt"

"github.com/fclairamb/solidping/server/internal/db/models"
)

// errResolveOpenNeedsDisable rejects resolveOpenIncidents without enabled:false.
// Listed in isCheckFieldValidationError so it is a 400.
var errResolveOpenNeedsDisable = errors.New("resolveOpenIncidents requires enabled: false")

// CheckDisableIncidentResolver closes the open incidents of a check that was
// just disabled by hand (spec 2026-10-23-01). It is an interface because the
// implementation lives in handlers/incidents, which this package cannot import.
type CheckDisableIncidentResolver interface {
ResolveOnCheckDisable(ctx context.Context, orgUID, checkUID string) ([]string, error)
}

// SetCheckDisableIncidentResolver wires the resolver. Every service serving a
// check write path (HTTP routes, MCP) must set it, or resolveOpenIncidents is
// silently ignored.
func (s *Service) SetCheckDisableIncidentResolver(resolver CheckDisableIncidentResolver) {
s.disableResolver = resolver
}

func validateResolveOpenIncidents(req *UpdateCheckRequest) error {
if req.ResolveOpenIncidents != nil && *req.ResolveOpenIncidents &&
(req.Enabled == nil || *req.Enabled) {
return errResolveOpenNeedsDisable
}

return nil
}

// resolveIncidentsOnDisable fires on any write that sets enabled:false with
// resolveOpenIncidents:true, not only on a true to false transition, so a retry
// after a failed resolve still closes them.
func (s *Service) resolveIncidentsOnDisable(
ctx context.Context, check *models.Check, req *UpdateCheckRequest,
) error {
if s.disableResolver == nil || req.ResolveOpenIncidents == nil || !*req.ResolveOpenIncidents ||
req.Enabled == nil || *req.Enabled {
return nil
}

if _, err := s.disableResolver.ResolveOnCheckDisable(ctx, check.OrganizationUID, check.UID); err != nil {
return fmt.Errorf("resolve open incidents: %w", err)
}

return nil
}
1 change: 1 addition & 0 deletions server/internal/handlers/checks/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -987,6 +987,7 @@ func isCheckFieldValidationError(err error) bool {
errors.Is(err, errFlapBackoffTooSmall) ||
errors.Is(err, errMaxRecoveryMultTooSmall) ||
errors.Is(err, errInvalidTraceroutePolicy) ||
errors.Is(err, errResolveOpenNeedsDisable) ||
errors.Is(err, errInvalidFailQuorum) ||
// Placement (spec 2026-09-25-06): a contradictory or unplaceable
// placement request is the caller's to fix.
Expand Down
14 changes: 14 additions & 0 deletions server/internal/handlers/checks/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,9 @@ type Service struct {
// degraded detection is turned off (spec 2026-09-24-08). nil = not wired,
// in which case nothing is resolved; see SetDegradedIncidentResolver.
degradedResolver DegradedIncidentResolver
// disableResolver closes a check's open incidents when it is disabled with
// resolveOpenIncidents (spec 2026-10-23-01). nil = not wired, nothing resolved.
disableResolver CheckDisableIncidentResolver
// checkDeletedNotifier tells whoever was paged that a deleted check's
// incident closed (spec 2026-10-08-02). nil = not wired: the incidents
// still resolve, silently. See SetCheckDeletedIncidentNotifier.
Expand Down Expand Up @@ -2004,6 +2007,10 @@ type UpdateCheckRequest struct {
RegionCount *int `json:"regionCount,omitempty"`
RegionPool *[]string `json:"regionPool,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
// ResolveOpenIncidents closes the check's open incidents with resolution
// type `check_disabled` in the same write. Only valid alongside
// `enabled: false` (spec 2026-10-23-01).
ResolveOpenIncidents *bool `json:"resolveOpenIncidents,omitempty"`
// Internal is decoded only so it can be refused — see
// ErrInternalFieldNotWritable (spec 2026-08-27-01).
Internal *bool `json:"internal,omitempty"`
Expand Down Expand Up @@ -2279,6 +2286,9 @@ func (s *Service) UpdateCheck(
return CheckResponse{}, coverErr
}
}
if validateErr := validateResolveOpenIncidents(req); validateErr != nil {
return CheckResponse{}, validateErr
}
if req.Enabled != nil {
update.Enabled = req.Enabled
}
Expand Down Expand Up @@ -2404,6 +2414,10 @@ func (s *Service) UpdateCheck(
return CheckResponse{}, resolveErr
}

if resolveErr := s.resolveIncidentsOnDisable(ctx, check, req); resolveErr != nil {
return CheckResponse{}, resolveErr
}

// Reconcile check jobs if regions, period, spread, enabled, or config
// changed (a regionSpread-only edit re-levels the per-region phases).
if regionsChanged || req.Period != nil || req.RegionSpread != nil ||
Expand Down
105 changes: 105 additions & 0 deletions server/internal/handlers/incidents/check_disabled.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package incidents

import (
"context"
"fmt"
"log/slog"

"github.com/fclairamb/solidping/server/internal/audit"
"github.com/fclairamb/solidping/server/internal/db/models"
)

// ResolveOnCheckDisable closes every active incident (outage and degraded) of a
// check the user just disabled with resolveOpenIncidents (spec 2026-10-23-01).
// Without it they would stay active forever: a disabled check has no jobs, so
// no result ever arrives to auto-resolve them.
//
// Incidents resolve with resolution_type = "check_disabled" and resolved_by =
// the caller when known, so history and notifications say the check was
// disabled rather than that it recovered. Returns the UIDs it resolved.
func (s *Service) ResolveOnCheckDisable(ctx context.Context, orgUID, checkUID string) ([]string, error) {
active, _, err := s.db.ListIncidents(ctx, &models.ListIncidentsFilter{
OrganizationUID: orgUID,
CheckUIDs: []string{checkUID},
States: []models.IncidentState{models.IncidentStateActive},
})
if err != nil {
return nil, fmt.Errorf("list active incidents: %w", err)
}

if len(active) == 0 {
return nil, nil
}

check, checkErr := s.db.GetCheck(ctx, orgUID, checkUID)
if checkErr != nil {
check = nil
}

actorUID := audit.ActorFromContext(ctx).UserUID
resolved := make([]string, 0, len(active))

for _, incident := range active {
if err := s.resolveForCheckDisable(ctx, orgUID, incident, check, actorUID); err != nil {
return resolved, err
}

resolved = append(resolved, incident.UID)
}

return resolved, nil
}

func (s *Service) resolveForCheckDisable(
ctx context.Context, orgUID string, incident *models.Incident, check *models.Check, actorUID string,
) error {
now := s.clock.Now()
state := models.IncidentStateResolved
resolutionType := models.ResolutionTypeCheckDisabled
update := models.IncidentUpdate{
State: &state,
ResolvedAt: &now,
ResolutionType: &resolutionType,
}

if actorUID != "" {
update.ResolvedBy = &actorUID
}

if err := s.db.UpdateIncident(ctx, incident.UID, &update); err != nil {
return fmt.Errorf("resolve incident %s: %w", incident.UID, err)
}

incident.State = state
incident.ResolvedAt = &now
incident.ResolutionType = &resolutionType
incident.ResolvedBy = update.ResolvedBy

// Cancel BEFORE emitting, or the sweep also drops the resolved
// notifications about to be queued.
s.cancelPendingNotifications(ctx, incident.UID, nil)

payload := models.JSONMap{
keyCheckUID: incident.CheckUID,
keyResolutionType: resolutionType,
keyResolvedAt: now,
keyDurationSeconds: int64(now.Sub(incident.StartedAt).Seconds()),
}
if check != nil {
payload[keyCheckSlug] = check.Slug
payload[keyCheckName] = check.Name
}

if actorUID != "" {
payload[payloadKeyActorUID] = actorUID
}

if err := s.emitEvent(ctx, orgUID, models.EventTypeIncidentResolved, incident, payload); err != nil {
slog.WarnContext(ctx, "Failed to emit check-disabled resolve event",
"incidentUid", incident.UID, "error", err)
}

s.publishResolved(ctx, incident)

return nil
}
Loading
Loading