Better handling of transient failures

e2e/status-page.spec.ts

@@ -2,9 +2,9 @@ import { test, expect } from "@playwright/test";
 
 test.use({ storageState: { cookies: [], origins: [] } });
 
-test("shows Sites heading", async ({ page }) => {
+test("shows Latest checks heading", async ({ page }) => {
   await page.goto("/");
-  await expect(page.getByRole("heading", { name: "Sites" })).toBeVisible();
+  await expect(page.getByRole("heading", { name: "Latest checks" })).toBeVisible();
 });
 
 test("shows seeded site card with name and URL", async ({ page }) => {

src/__tests__/check-runner.test.ts

@@ -1,10 +1,14 @@
 import { describe, test, expect, vi, beforeEach } from 'vitest'
 import { NextRequest } from 'next/server'
 
-// Mock the checker module
-vi.mock('@/lib/checker', () => ({
-  checkSite: vi.fn(),
-}))
+// Mock the checker module — keep real isSoftFailure, mock checkSite
+vi.mock('@/lib/checker', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@/lib/checker')>()
+  return {
+    ...actual,
+    checkSite: vi.fn(),
+  }
+})
 
 // Mock the notifications module
 const mockSendIncidentAlert = vi.fn()
@@ -86,7 +90,7 @@ describe('POST /api/checks/run', () => {
     expect(body).toEqual({ message: 'No sites to check', checks: 0, incidents: 0 })
   })
 
-  test('records checks and opens incidents for failures, sends email', async () => {
+  test('hard failure (HTTP 500) creates incident immediately, sends email', async () => {
     const site = { id: 'site-1', name: 'Test', url: 'https://example.com' }
 
     mockFrom.mockImplementation((table: string) => {
@@ -134,8 +138,8 @@ describe('POST /api/checks/run', () => {
 
     vi.mocked(checkSite).mockResolvedValue({
       status: 'failure',
-      statusCode: 503,
-      error: 'HTTP 503',
+      statusCode: 500,
+      error: 'HTTP 500',
     })
 
     const request = new NextRequest('http://localhost/api/checks/run', {
@@ -150,12 +154,211 @@ describe('POST /api/checks/run', () => {
     expect(mockSendIncidentAlert).toHaveBeenCalledWith({
       siteName: 'Test',
       siteUrl: 'https://example.com',
-      error: 'HTTP 503',
+      error: 'HTTP 500',
       incidentId: 'inc-1',
       contactEmails: ['alice@example.com'],
     })
   })
 
+  test('single soft failure (HTTP 503) does not create incident', async () => {
+    const site = { id: 'site-1', name: 'Test', url: 'https://example.com' }
+
+    mockFrom.mockImplementation((table: string) => {
+      if (table === 'sites') {
+        return { select: () => ({ data: [site], error: null }) }
+      }
+      if (table === 'checks') {
+        return {
+          insert: () => ({
+            select: () => ({
+              single: () => ({
+                data: { id: 'check-1', site_id: 'site-1', status: 'failure' },
+              }),
+            }),
+          }),
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                gte: () => ({
+                  data: [{ status_code: 503, error: 'HTTP 503' }],
+                }),
+              }),
+            }),
+          }),
+        }
+      }
+      if (table === 'incidents') {
+        return {
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                limit: () => ({
+                  single: () => ({ data: null }),
+                }),
+              }),
+            }),
+          }),
+        }
+      }
+      return {}
+    })
+
+    vi.mocked(checkSite).mockResolvedValue({
+      status: 'failure',
+      statusCode: 503,
+      error: 'HTTP 503',
+    })
+
+    const request = new NextRequest('http://localhost/api/checks/run', {
+      method: 'POST',
+      headers: { authorization: 'Bearer test-secret' },
+    })
+
+    const response = await POST(request)
+    const body = await response.json()
+    expect(body.incidents).toBe(0)
+    expect(mockSendIncidentAlert).not.toHaveBeenCalled()
+  })
+
+  test('three soft failures within an hour creates incident', async () => {
+    const site = { id: 'site-1', name: 'Test', url: 'https://example.com' }
+
+    mockFrom.mockImplementation((table: string) => {
+      if (table === 'sites') {
+        return { select: () => ({ data: [site], error: null }) }
+      }
+      if (table === 'checks') {
+        return {
+          insert: () => ({
+            select: () => ({
+              single: () => ({
+                data: { id: 'check-3', site_id: 'site-1', status: 'failure' },
+              }),
+            }),
+          }),
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                gte: () => ({
+                  data: [
+                    { status_code: 503, error: 'HTTP 503' },
+                    { status_code: 502, error: 'HTTP 502' },
+                    { status_code: 503, error: 'HTTP 503' },
+                  ],
+                }),
+              }),
+            }),
+          }),
+        }
+      }
+      if (table === 'incidents') {
+        return {
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                limit: () => ({
+                  single: () => ({ data: null }),
+                }),
+              }),
+            }),
+          }),
+          insert: () => ({
+            select: () => ({
+              single: () => ({ data: { id: 'inc-1' } }),
+            }),
+          }),
+        }
+      }
+      if (table === 'contacts') {
+        return {
+          select: () => ({
+            data: [{ email: 'alice@example.com' }],
+          }),
+        }
+      }
+      return {}
+    })
+
+    vi.mocked(checkSite).mockResolvedValue({
+      status: 'failure',
+      statusCode: 503,
+      error: 'HTTP 503',
+    })
+
+    const request = new NextRequest('http://localhost/api/checks/run', {
+      method: 'POST',
+      headers: { authorization: 'Bearer test-secret' },
+    })
+
+    const response = await POST(request)
+    const body = await response.json()
+    expect(body.incidents).toBe(1)
+    expect(mockSendIncidentAlert).toHaveBeenCalled()
+  })
+
+  test('two soft failures within an hour does not create incident', async () => {
+    const site = { id: 'site-1', name: 'Test', url: 'https://example.com' }
+
+    mockFrom.mockImplementation((table: string) => {
+      if (table === 'sites') {
+        return { select: () => ({ data: [site], error: null }) }
+      }
+      if (table === 'checks') {
+        return {
+          insert: () => ({
+            select: () => ({
+              single: () => ({
+                data: { id: 'check-2', site_id: 'site-1', status: 'failure' },
+              }),
+            }),
+          }),
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                gte: () => ({
+                  data: [
+                    { status_code: 503, error: 'HTTP 503' },
+                    { status_code: 504, error: 'HTTP 504' },
+                  ],
+                }),
+              }),
+            }),
+          }),
+        }
+      }
+      if (table === 'incidents') {
+        return {
+          select: () => ({
+            eq: () => ({
+              eq: () => ({
+                limit: () => ({
+                  single: () => ({ data: null }),
+                }),
+              }),
+            }),
+          }),
+        }
+      }
+      return {}
+    })
+
+    vi.mocked(checkSite).mockResolvedValue({
+      status: 'failure',
+      statusCode: 503,
+      error: 'HTTP 503',
+    })
+
+    const request = new NextRequest('http://localhost/api/checks/run', {
+      method: 'POST',
+      headers: { authorization: 'Bearer test-secret' },
+    })
+
+    const response = await POST(request)
+    const body = await response.json()
+    expect(body.incidents).toBe(0)
+    expect(mockSendIncidentAlert).not.toHaveBeenCalled()
+  })
+
   test('records checks without creating incidents for successful checks', async () => {
     const site = { id: 'site-1', name: 'Test', url: 'https://example.com' }
     const mockInsert = vi.fn().mockReturnValue({

src/__tests__/checker.test.ts

@@ -1,5 +1,5 @@
 import { describe, test, expect, vi, beforeEach } from 'vitest'
-import { checkSite } from '@/lib/checker'
+import { checkSite, isSoftFailure } from '@/lib/checker'
 
 describe('checkSite', () => {
   beforeEach(() => {
@@ -63,3 +63,38 @@ describe('checkSite', () => {
     })
   })
 })
+
+describe('isSoftFailure', () => {
+  test('HTTP 502 is a soft failure', () => {
+    expect(isSoftFailure(502, 'HTTP 502')).toBe(true)
+  })
+
+  test('HTTP 503 is a soft failure', () => {
+    expect(isSoftFailure(503, 'HTTP 503')).toBe(true)
+  })
+
+  test('HTTP 504 is a soft failure', () => {
+    expect(isSoftFailure(504, 'HTTP 504')).toBe(true)
+  })
+
+  test('Connection timeout is a soft failure', () => {
+    expect(isSoftFailure(null, 'Connection timeout')).toBe(true)
+  })
+
+  test('HTTP 500 is a hard failure', () => {
+    expect(isSoftFailure(500, 'HTTP 500')).toBe(false)
+  })
+
+  test('HTTP 404 is a hard failure', () => {
+    expect(isSoftFailure(404, 'HTTP 404')).toBe(false)
+  })
+
+  test('DNS resolution failure is a hard failure', () => {
+    expect(isSoftFailure(null, 'getaddrinfo ENOTFOUND example.com')).toBe(false)
+  })
+
+  test('SSL error is a hard failure', () => {
+    expect(isSoftFailure(null, 'unable to verify the first certificate')).toBe(false)
+  })
+})
+

src/app/api/checks/run/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server'
 import { createAdminClient } from '@/lib/supabase/admin'
-import { checkSite } from '@/lib/checker'
+import { checkSite, isSoftFailure } from '@/lib/checker'
 import { sendIncidentAlert } from '@/lib/notifications'
 
 export const maxDuration = 60
@@ -60,29 +60,51 @@ async function handleCheckRun(request: NextRequest) {
         .single()
 
       if (!openIncident) {
-        const { data: incident } = await supabase
-          .from('incidents')
-          .insert({
-            site_id: site.id,
-            check_id: check.id,
-            status: 'open',
-          })
-          .select('id')
-          .single()
-        newIncidents++
-
-        if (incident) {
-          const { data: contacts } = await supabase
-            .from('contacts')
-            .select('email')
-
-          await sendIncidentAlert({
-            siteName: site.name,
-            siteUrl: site.url,
-            error,
-            incidentId: incident.id,
-            contactEmails: (contacts ?? []).map((c: { email: string }) => c.email),
-          })
+        const soft = isSoftFailure(statusCode, error)
+        let shouldCreateIncident = !soft // Hard failures always create incidents
+
+        if (soft) {
+          // Soft failures only create incidents when there are 3+ within the last hour
+          const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString()
+          const { data: recentFailures } = await supabase
+            .from('checks')
+            .select('status_code, error')
+            .eq('site_id', site.id)
+            .eq('status', 'failure')
+            .gte('checked_at', oneHourAgo)
+
+          const softFailureCount = (recentFailures ?? []).filter(
+            (c: { status_code: number | null; error: string | null }) =>
+              isSoftFailure(c.status_code, c.error)
+          ).length
+          shouldCreateIncident = softFailureCount >= 3
+        }
+
+        if (shouldCreateIncident) {
+          const { data: incident } = await supabase
+            .from('incidents')
+            .insert({
+              site_id: site.id,
+              check_id: check.id,
+              status: 'open',
+            })
+            .select('id')
+            .single()
+          newIncidents++
+
+          if (incident) {
+            const { data: contacts } = await supabase
+              .from('contacts')
+              .select('email')
+
+            await sendIncidentAlert({
+              siteName: site.name,
+              siteUrl: site.url,
+              error,
+              incidentId: incident.id,
+              contactEmails: (contacts ?? []).map((c: { email: string }) => c.email),
+            })
+          }
         }
       }
     }